python3爬取国家统计局,区域城乡代码
程序员文章站
2022-03-02 21:08:13
...
# -*- coding: utf-8 -*-
# author:zjp
"""
通过国家统计局官网获取中国2017年所有城市数据
2017年统计用区划代码和城乡划分代码(截止2017年10月31日)
http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/
"""
import re
import requests
import time
import operator
from functools import reduce
save_route = 'F://python/China_Province_2017.txt' # 数据储存路径
results2 = []
results3 = []
results4 = []
results5 = []
Dates1 = []
kv = {'user-agent': 'Mozilla/5.0'}
n = 0
url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/index.html'
r = requests.get(url, headers=kv)
r.raise_for_status()
r.encoding = r.apparent_encoding
pattern = re.compile("<a href='(.*?)'>(.*?)<") # 正则表达式
result1 = list(set(re.findall(pattern, r.text))) # 从主页面获取子页面的html
print('result1')
# print(result1)
i2 = 0
for i2 in range(len(result1)):
try:
url2a = result1[i2][0]
address1 = result1[i2][1] # 一级地址
i2 += 1
url2 = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/' + url2a
# http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/44.html
# print(url2)
# print(address1)
r2 = requests.get(url2, headers=kv)
r2.raise_for_status()
r2.encoding = r2.apparent_encoding
pattern2 = re.compile("<a href='(.*?)'>(.*?)<") # 正则表达式提取目标字段
result2 = list(set(re.findall(pattern2, r2.text)))
# print(result2)
result2a = []
for i2a in result2: # 爬取的城市信息和城市代码混在一起,需要将代码清除
if '0' in i2a[1]:
n += 1
else:
result2a.append(i2a)
print('result2a')
# print(result2a)
except:
print('错误')
with open(save_route, 'a', encoding='utf-8')as f:
f.write('一级错误 一级错误 一级错误 一级错误')
f.write('\n')
f.close()
time.sleep(10)
continue
i3 = 0
for i3 in range(len(result2a)):
try:
url3a = result2a[i3][0]
address2 = result2a[i3][1] # 二级地址
i3 += 1
url3 = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/' + url3a
# http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/34/3401.html
r3 = requests.get(url3, headers=kv)
r3.raise_for_status()
r3.encoding = r3.apparent_encoding
pattern3 = re.compile("<a href='(.*?)'>(.*?)<")
result3 = list(set(re.findall(pattern3, r3.text)))
# print(result3)
result3a = []
for i3a in result3:
if '0' in i3a[1]:
n += 1
else:
result3a.append(i3a)
print('result3a')
except:
print('错误')
with open(save_route, 'a', encoding='utf-8')as f:
f.write('二级错误 二级错误 二级错误 二级错误')
f.write('\n')
f.close()
time.sleep(10)
continue
# print(result3a)
i4 = 0
for i4 in range(len(result3a)):
try:
url4a = result3a[i4][0]
address3 = result3a[i4][1] # 二级地址
i4 += 1
url4 = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/' + url4a[3:5] + '/' + url4a
# print(url4)
# http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/41/04/410481.html
r4 = requests.get(url4, headers=kv)
r4.raise_for_status()
r4.encoding = r4.apparent_encoding
pattern4 = re.compile("<a href='(.*?)'>(.*?)<")
result4 = list(set(re.findall(pattern3, r4.text)))
# print(result4)
result4a = []
for i4a in result4:
if '0' in i4a[1]:
n += 1
else:
result4a.append(i4a)
print('result4a')
except:
print('错误')
with open(save_route, 'a', encoding='utf-8')as f:
f.write('三级错误 三级错误 三级错误 三级错误')
f.write('\n')
f.close()
time.sleep(10)
continue
# print(result4a)
results5 = []
i5 = 0
for i5 in range(len(result4a)):
try:
address4 = result4a[i5][1]
i5 += 1
address = str(address1) + ' ' + str(address2) + ' ' + str(address3) + ' ' + str(address4)
print(address)
with open(save_route, 'a', encoding='utf-8')as f:
f.write(address)
f.write('\n')
f.close()
except:
print('错误')
with open(save_route, 'a', encoding='utf-8')as f:
f.write('四级错误 四级错误 四级错误 四级错误')
f.write('\n')
f.close()
time.sleep(10)
continue
print('well_done')
原文地址:https://blog.csdn.net/mg_aping/article/details/82992913
推荐阅读
-
python3实现爬取淘宝美食代码分享
-
Python3爬取英雄联盟英雄皮肤大图实例代码
-
python3爬取咪咕音乐榜信息(附源代码)
-
python3爬取淘宝信息代码分析
-
python3实现爬取淘宝美食代码分享
-
Python3爬取英雄联盟英雄皮肤大图实例代码
-
Python3爬取国家统计局官网2019年全国所有城市(2020年更新)
-
Python爬虫练习五:爬取 2017年统计用区划代码和城乡划分代码(附代码与全部数据)
-
python爬取2017年统计用区划代码和城乡划分代码(截止2017年10月31日)
-
python3使用Selenium+Chrome+BeautifulSoup爬取国家统计局数据(亲测成功脚本)