由于疫情影响,导致物流、原料、人工等成本大幅增加,部分芯片原厂停产、减产,导致芯片出厂价上调。另外一部分芯片经销商人为囤货或故意抬高价格导致芯片价格走势一路上涨。如果可以及时跟踪各芯片厂家的官网或交易平台,实时关注库存情况及时下单,可以减少中间环节,降低采购成本。通过爬虫代理实现多并发、低延迟的网络请求,及时分析海量商品动态,可以满足芯片业务要求,示例如下:
#! -*- encoding:utf-8 -*-
import urllib2
import random
import httplib
class HTTPSConnection(httplib.HTTPSConnection):
def set_tunnel(self, host, port=None, headers=None):
httplib.HTTPSConnection.set_tunnel(self, host, port, headers)
if hasattr(self, 'proxy_tunnel'):
self._tunnel_headers['Proxy-Tunnel'] = self.proxy_tunnel
class HTTPSHandler(urllib2.HTTPSHandler):
def https_open(self, req):
return urllib2.HTTPSHandler.do_open(self, HTTPSConnection, req, context=self._context)
# 要访问的目标页面
targetUrlList = [
"https://www.ti.com",
"https://www.1688.com/",
]
# 代理服务器(产品官网 www.16yun.cn)
proxyHost = "t.16yun.cn"
proxyPort = "31111"
# 代理验证信息
proxyUser = "username"
proxyPass = "password"
proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {
"host": proxyHost,
"port": proxyPort,
"user": proxyUser,
"pass": proxyPass,
}
# 设置 http和https访问都是用HTTP代理
proxies = {
"http": proxyMeta,
"https": proxyMeta,
}
# 设置IP切换头
tunnel = random.randint(1, 10000)
headers = {"Proxy-Tunnel": str(tunnel)}
HTTPSConnection.proxy_tunnel = tunnel
proxy = urllib2.ProxyHandler(proxies)
opener = urllib2.build_opener(proxy, HTTPSHandler)
urllib2.install_opener(opener)
# 访问三次网站,使用相同的tunnel标志,均能够保持相同的外网IP
for i in range(2):
for url in targetUrlList:
r = urllib2.Request(url)
print(urllib2.urlopen(r).read())