我的第一个爬虫，爬取北京地区短租房信息

# 导入程序所需要的库。
import requests
from bs4 import beautifulsoup
import time

# 加入请求头伪装成浏览器
headers = {
    #通过chrome浏览器复制user-agent
    'user-agent': 'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/69.0.3497.100 safari/537.36'
}


# 定义判断用户性别的函数
def judgment_sex(class_name):
    if class_name == ['member_ico1']:
        return '女'
    else:
        return '男'


# 获取详细页url函数
def get_links(url):
    try:
        wb_date = requests.get(url, headers)
    except connectionabortederror:
        print('拒绝连接')
    soup = beautifulsoup(wb_date.text, 'lxml')
    links = soup.select('#page_list > ul > li > a')
    for link in links:
        herf = link.get("href")
        get_info(herf)


# 获取网页信息函数
def get_info(url):
    wb_date = requests.get(url, headers)
    soup = beautifulsoup(wb_date.text, 'lxml')
    #通过浏览器copy selector
    tittles = soup.select('div.pho_info > h4')
    addresses = soup.select('span.pr5')
    prises = soup.select('#pricepart > div.day_l > span')
    images = soup.select('#floatrightbox > div.js_box.clearfix > div.member_pic > a > img')
    names = soup.select('#floatrightbox > div.js_box.clearfix > div.w_240 > h6 > a')
    sexs = soup.select('#floatrightbox > div.js_box.clearfix > div.member_pic > div')
    for tittle, address, prise, image, name, sex in zip(tittles, addresses, prises, images, names, sexs):
        date = {
            'tittle': tittle.get_text().strip(),
            'address': address.get_text().strip(),
            'price': prise.get_text(),
            'image': image.get("src"),
            'name': name.get_text(),
            'sex': judgment_sex(sex.get("class"))
        }
        print(date)


if __name__ == '__main__':

    urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(number) for number in range(1, 14)]
    for single_url in urls:
        get_links(single_url)
        #休眠十秒，防止被封ip
        time.sleep(10)

        #缺点：缺少ip管理，采用休眠方法，效率低