Python进阶教程：网络编程与数据抓取

1
import urllib.request
2
import urllib.parse
3

4
# GET 请求
5
url = "https://httpbin.org/get"
6
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
7
with urllib.request.urlopen(req, timeout=10) as resp:
8
    print(resp.status)      # 200
9
    print(resp.read().decode("utf-8")[:200])
10

11
# POST 请求
12
data = urllib.parse.urlencode({"name": "Alice", "age": 20}).encode()
13
req = urllib.request.Request("https://httpbin.org/post", data=data)
14
with urllib.request.urlopen(req) as resp:
15
    print(resp.read().decode("utf-8")[:200])

1
import requests
2

3
# GET 请求
4
resp = requests.get("https://httpbin.org/get", params={"q": "python"}, timeout=10)
5
print(resp.status_code)   # 200
6
print(resp.json())        # 自动解析 JSON
7

8
# POST 请求
9
resp = requests.post("https://httpbin.org/post", json={"name": "Alice"})
10
print(resp.json())
11

12
# 自定义请求头（模拟浏览器）
13
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
14
resp = requests.get("https://httpbin.org/headers", headers=headers)
15

16
# 下载文件
17
resp = requests.get("https://httpbin.org/image/png", stream=True)
18
with open("image.png", "wb") as f:
19
    for chunk in resp.iter_content(chunk_size=8192):
20
        f.write(chunk)

1
from bs4 import BeautifulSoup
2
import requests
3

4
html = """
5
<html><body>
6
  <h1>Python 教程</h1>
7
  <div class="article">
8
    <a href="/p1">第一篇</a>
9
    <a href="/p2">第二篇</a>
10
  </div>
11
</body></html>
12
"""
13

14
soup = BeautifulSoup(html, "html.parser")
15

16
# 获取标题
17
print(soup.h1.text)                # Python 教程
18

19
# 按 class 查找
20
div = soup.find("div", class_="article")
21

22
# 查找所有链接
23
for a in div.find_all("a"):
24
    print(a.text, a["href"])
25
# 第一篇 /p1
26
# 第二篇 /p2

1
import requests
2
from bs4 import BeautifulSoup
3

4
def fetch_links(url):
5
    """抓取页面中所有链接及其文本"""
6
    try:
7
        headers = {"User-Agent": "Mozilla/5.0"}
8
        resp = requests.get(url, headers=headers, timeout=10)
9
        resp.raise_for_status()  # 非 200 会抛出异常
10

11
        soup = BeautifulSoup(resp.text, "html.parser")
12
        links = []
13
        for a in soup.find_all("a", href=True):
14
            text = a.text.strip() or "(无文本)"
15
            links.append((text[:30], a["href"]))
16
        return links
17
    except requests.RequestException as e:
18
        print(f"请求失败：{e}")
19
        return []
20

21
# 使用示例
22
url = "https://example.com"
23
for text, href in fetch_links(url)[:10]:
24
    print(f"{text} -> {href}")

1
import time
2
import requests
3

4
urls = ["https://httpbin.org/get"] * 5
5
for url in urls:
6
    resp = requests.get(url, timeout=10)
7
    print(resp.status_code)
8
    time.sleep(2)  # 每 2 秒请求一次，礼貌抓取

1
import requests
2
import json
3

4
def call_api(url, params=None):
5
    resp = requests.get(url, params=params, timeout=10)
6
    if resp.status_code == 200:
7
        return resp.json()
8
    else:
9
        print(f"API 返回错误：{resp.status_code}")
10
        return None
11

12
# 调用公开 API 获取天气信息（示例）
13
data = call_api("https://httpbin.org/json")
14
if data:
15
    print(json.dumps(data, ensure_ascii=False, indent=2))