下载网页图片到本地并替换网页内的图片链接

1
2
3
4
5
6

# 导包
import requests
import os
import pathlib
from lxml import etree
import re

1
2
3
4
5
6

base_dir = pathlib.Path(r"D:\Projects\book_hands_on_machine_learning")

file_path = base_dir / "book.html"

with open(file_path, 'r', encoding='utf-8') as f:
    html_content = f.read()

1
2
3
4
5
6

def parse_xpath(html, xpath_stat):
    parse_html = etree.HTML(html)
    result_list = parse_html.xpath(xpath_stat)
    return result_list

img_xpath_list = parse_xpath(html_content, '//img')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

def get_img_list(xpath_list):
    img_url_list = []
    for item in xpath_list:
        url = item.xpath('./@src')[0]
        img_url_list.append(url)

    # 链接去重并保留原顺序
    img_url_list_new = list(set(img_url_list))
    img_url_list_new.sort(key=img_url_list.index)

    return img_url_list_new

img_url_list = get_img_list(img_xpath_list)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

def save_img(img_urls: list):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',
        'Cookie': '填自己的'
    }
    url_file_path_tuple_list = []
    img_list_length = len(img_urls)

    img_dir = base_dir / 'imgs'
    if not os.path.exists(img_dir):
        os.makedirs(img_dir)

    for i, url in enumerate(img_urls):
        img_name = url.split('/')[-1] + '.jpg'
        img_path = img_dir / img_name
        res = requests.get(url=url, headers=headers).content
        with open(img_path, 'wb') as f:
            f.write(res)

        print(f"进度: {(i + 1) / img_list_length * 100:.2f}% 图片 {url} 下载成功.")
        url_file_path_tuple_list.append((url, os.path.relpath(img_path, base_dir)))

    return url_file_path_tuple_list

url_file_path_tuple_list = save_img(img_url_list)

1
2
3
4
5
6
7

def replace_img_url(html_content, url_file_path_tuple_list):
    for url, img_path in url_file_path_tuple_list:
        html_content = html_content.replace(f'src="{url}"', f'src="{img_path}"')

    return html_content

new_parsed_html = replace_img_url(html_content, url_file_path_tuple_list)

1
2
3
4
5

new_parsed_html = re.sub(r'href="http://popImage\?src=.*?"', 'href="#"', new_parsed_html)

new_parsed_html = re.sub(r'<a .*?>', '', new_parsed_html)

new_parsed_html = re.sub(r'</a>', '', new_parsed_html)

1
2

with open(base_dir / 'result.html', 'w', encoding="utf-8") as f:
    f.write(new_parsed_html)