-
Notifications
You must be signed in to change notification settings - Fork 1
/
addrLoop.py
34 lines (25 loc) · 1.1 KB
/
addrLoop.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
#This file loops over all city codes set by n range and extracts address data from each entry
from bs4 import BeautifulSoup
import requests
import csv
addr_list = []
#loop over different cities
for n in range(1,10):
base_url = "https://www.dianping.com/search/keyword/"+str(n)+"/0_%E7%BB%98%E6%9C%AC%E9%A6%86/"
if n<4:
for page in range(1, 13):
print ("Code: " + str(n) + " Current page: " + str(page))
url = base_url + 'p' + str(page)
response = requests.get(url)
data = response.text
soup = BeautifulSoup(data, "html.parser")
addr_list += soup.find_all('span',{'class':'addr'})
if n>3 and n!=20 and n!=51 and n!=189 and n!=190 and n!=191:
for page in range(1, 5):
print ("Code: " + str(n) + " Current page: " + str(page))
url = base_url + 'p' + str(page)
response = requests.get(url)
data = response.text
soup = BeautifulSoup(data, "html.parser")
addr_list += soup.find_all('span',{'class':'addr'})
print(addr_list)