Create a simple sitemap generator with Python

1pip install requests
2

1pip install beautifulsoup4
2

1pip install html5lib
2

1def clean(a_eles):
2    links = []
3    skip_links = []
4    for a in a_eles:
5        link = a['href']
6        if link.startswith('#') or link.startswith('mailto:') or link == '/':
7            skip_links.append(link)
8            continue
9
10        if link.startswith('/'):
11            link = '{}{}'.format(base_url, link)
12
13        if link.startswith('http://') != True and link.startswith('https://') != True:
14            link = '{}/{}'.format(base_url, link)
15
16        if link.startswith(base_url) is False:
17            continue
18
19        if link not in links:
20            links.append(link)
21
22    return [links, skip_links]
23

1def get_next_scan_urls(urls):
2    links = []
3    for u in urls:
4        if u not in scanned:
5            links.append(u)
6    return links
7

1def scan(url):
2    if url not in scanned:
3        print('Scan url: {}'.format(url))
4        scanned.append(url)
5        data = requests.get(url)
6        soup = BeautifulSoup(data.text, 'html5lib')
7        a_eles = soup.find_all('a', href=True)
8        links, skip_links = clean(a_eles)
9
10        next_scan_urls = get_next_scan_urls(links)
11        print('Count next scan: {}'.format(len(next_scan_urls)))
12        if len(next_scan_urls) != 0:
13            for l in next_scan_urls:
14                scan(l)
15    return scanned
16

1def main():
2    links = scan(website)
3
4    urls = ''
5    for l in links:
6        urls += f"""
7    <url>
8      <loc>{l}</loc>
9      <lastmod>2022-07-27T02:24:08.242Z</lastmod>
10      <priority>0.6</priority>
11    </url>
12        """
13
14    xml = f"""
15<?xml version="1.0" encoding="UTF-8"?>
16<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
17    {urls}
18</urlset>
19    """
20
21    f = open('sitemap.xml', 'w')
22    f.write(xml)
23    f.close()
24

1#!/usr/bin/env python
2
3import requests
4from bs4 import BeautifulSoup
5
6website = 'https://www.khmernokor.com'
7base_url = website
8if website.endswith('/'):
9    base_url = website[:-1]
10
11scanned = []
12
13
14def clean(a_eles):
15    links = []
16    skip_links = []
17    for a in a_eles:
18        link = a['href']
19        if link.startswith('#') or link.startswith('mailto:') or link == '/':
20            skip_links.append(link)
21            continue
22
23        if link.startswith('/'):
24            link = '{}{}'.format(base_url, link)
25
26        if link.startswith('http://') != True and link.startswith('https://') != True:
27            link = '{}/{}'.format(base_url, link)
28
29        if link.startswith(base_url) is False:
30            continue
31
32        if link not in links:
33            links.append(link)
34
35    return [links, skip_links]
36
37
38def get_next_scan_urls(urls):
39    links = []
40    for u in urls:
41        if u not in scanned:
42            links.append(u)
43    return links
44
45
46def scan(url):
47    if url not in scanned:
48        print('Scan url: {}'.format(url))
49        scanned.append(url)
50        data = requests.get(url)
51        soup = BeautifulSoup(data.text, 'html5lib')
52        a_eles = soup.find_all('a', href=True)
53        links, skip_links = clean(a_eles)
54
55        next_scan_urls = get_next_scan_urls(links)
56        print('Count next scan: {}'.format(len(next_scan_urls)))
57        if len(next_scan_urls) != 0:
58            for l in next_scan_urls:
59                scan(l)
60    return scanned
61
62
63def main():
64    links = scan(website)
65
66    urls = ''
67    for l in links:
68        urls += f"""
69    <url>
70      <loc>{l}</loc>
71      <lastmod>2022-07-27T02:24:08.242Z</lastmod>
72      <priority>0.6</priority>
73    </url>
74        """
75
76    xml = f"""
77<?xml version="1.0" encoding="UTF-8"?>
78<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
79    {urls}
80</urlset>
81    """
82
83    f = open('sitemap.xml', 'w')
84    f.write(xml)
85    f.close()
86
87
88if __name__ == '__main__':
89    main()
90

1<?xml version="1.0" encoding="UTF-8"?>
2<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
3    <url>
4      <loc>https://www.khmernokor.com</loc>
5      <lastmod>2022-07-27T02:24:08.242Z</lastmod>
6      <priority>0.6</priority>
7    </url>
8
9    <url>
10      <loc>https://www.khmernokor.com/question-answers/jiwn5gvqpu</loc>
11      <lastmod>2022-07-27T02:24:08.242Z</lastmod>
12      <priority>0.6</priority>
13    </url>
14
15    <url>
16      <loc>https://www.khmernokor.com/bun98</loc>
17      <lastmod>2022-07-27T02:24:08.242Z</lastmod>
18      <priority>0.6</priority>
19    </url>
20
21    <url>
22      <loc>https://www.khmernokor.com/yuravandy</loc>
23      <lastmod>2022-07-27T02:24:08.242Z</lastmod>
24      <priority>0.6</priority>
25    </url>
26    // ....
27</urlset>
28