import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

visited=set()
files=[]

def crawl(url):
    if url in visited:
        return
    visited.add(url)

    r=requests.get(url,timeout=30)
    soup=BeautifulSoup(r.text,"html.parser")

    for a in soup.find_all("a"):
        href=a.get("href")
        if not href:
            continue

        full=urljoin(url,href)

        if href.endswith("/"):
            if href not in ("../","./"):
                crawl(full)
        else:
            files.append(full)

crawl("https://vmware.digiboy.ir/")

with open("urls.txt","w") as f:
    f.write("\n".join(files))