爬虫测试

胖虎发表于 2018-3-15 16:31:32

#coding=utf-8

__doc__ = '''''
使用requests来简单的爬取图片，请求使用Referer，否则爬取不到正确的结果
'''

CHUNK = 1024

import requests
import re
import os

def download_images(x, y):
URL_SEGMENT = '{0}/{1}'.format(x,y)
URL_FORMAT = 'http://img.zngirls.com/gallery/%s/{0:03d}.jpg' % (URL_SEGMENT)
URL_FORMAT0 = 'http://img.zngirls.com/gallery/%s/{0}.jpg' % (URL_SEGMENT)

i=0
numbers = re.compile('\\d+/\\d+')
rl = numbers.findall(URL_FORMAT0)
if not rl:
   return
dirname = rl.replace('/','-')
if not os.path.isdir(dirname):
   os.makedirs(dirname)

while True:
   url = ''
   if i==0:
         url = URL_FORMAT0.format(i)
   else:
         url = URL_FORMAT.format(i)

   print('url=', url)
   res = requests.get(url, headers={'Referer':'http://www.zngirls.com/g/13080/2.html',}, stream=True)
   if res.status_code != 200:
         break

   filename = os.path.join(dirname,'{0:03d}.jpg'.format(i))
   with open(filename, mode='wb') as f:
         for chunk in res.iter_content(CHUNK):
            f.write(chunk)

   i += 1

def main():
download_images(21363, 18304)

if __name__ == '__main__':
main()

海海豚 发表于 2018-3-16 13:32:14

谢谢分享！

libingyu135 发表于 2018-4-25 16:42:58

6666

梦想家 发表于 2018-5-8 10:19:48

:victory:

Miss_love 发表于 2018-5-8 13:28:43

:handshake

梦想家 发表于 2018-5-8 13:44:26

:victory:

页: [1]

51Testing软件测试论坛 's Archiver

爬虫测试