Merge pull request #17 from mxrch/light-update

1.1.0 update

gitfive/lib/commits.py

@@ -1,8 +1,5 @@
-import httpx
 import trio
 from bs4 import BeautifulSoup
-import imagehash
-from PIL import Image
 from alive_progress import alive_bar
 
 
@@ -15,22 +12,31 @@ async def fetch_avatar(runner: GitfiveRunner, email: str, avatar_link: str, user
                         out: Dict[str, str|bool], check_only: bool):
     async with runner.limiters["commits_fetch_avatar"]:
         is_target = (username.lower() == runner.target.username.lower())
-        full_name = await github.fetch_profile_name(runner, username)
-        _name_str = ""
-        if full_name:
-            _name_str = f" [{full_name}]"
-        if is_target:
-            if check_only:
-                runner.rc.print(f"[+] [Target's email] 🐱 {email} -> @{username}{_name_str}", style="cyan")
-            else:
+        if check_only:            
+            if is_target:
+                runner.rc.print(f"[+] [Target's email] 🐱 {email} -> @{username}", style="cyan")
+
+            out[email] = {
+                "avatar": avatar_link,
+                "username": username,
+                "is_target": is_target
+            }
+        else:
+            full_name = await github.fetch_profile_name(runner, username)
+            _name_str = ""
+            if full_name:
+                _name_str = f" [{full_name}]"
+
+            if is_target:
                 runner.rc.print(f"[+] [TARGET FOUND] 🐱 {email} -> @{username}{_name_str}", style="green bold")
-        elif not check_only:
-            runner.rc.print(f"[+] 🐱 {email} -> @{username}{_name_str}")
-        out[email] = {
-            "avatar": avatar_link,
-            "username": username,
-            "full_name": full_name,
-            "is_target": is_target
+            else:
+                runner.rc.print(f"[+] 🐱 {email} -> @{username}{_name_str}")
+
+            out[email] = {
+                "avatar": avatar_link,
+                "full_name": full_name,
+                "username": username,
+                "is_target": is_target
             }
 
 async def fetch_commits(runner: GitfiveRunner, repo_name: str, emails_index: Dict[str, str],
@@ -73,7 +79,7 @@ async def scrape(runner: GitfiveRunner, repo_name: str, emails_index: Dict[str,
         exit("Empty repository.")
 
     if last_hash_trigger in req.text:
-        total = int(body.select('div.Box-header strong')[0].text.replace(',', ''))
+        _, total = await get_commits_count(runner, raw_body=req.text)
         last_hash = [x for x in body.select('a') if x.text.lower() == "permalink"][0].attrs['href'].split('/')[-1]
     else:
         exit("Couldn't fetch the last hash.")

gitfive/lib/domain_finder.py

@@ -14,12 +14,15 @@ def guess_custom_domain(runner: GitfiveRunner):
     hunter = None
 
     # Google
-    if company != "google": # googlesearch doesn't return Google.com when searching "google"
-        for url in search(company):
-            if ("facebook" not in company and "facebook.com" in url) or ("twitter" not in company and "twitter.com" in url) :
-                continue
-            google = extract_domain(url)
-            break
+    try:
+        if company != "google": # googlesearch doesn't return Google.com when searching "google"
+            for url in search(company):
+                if ("facebook" not in company and "facebook.com" in url) or ("twitter" not in company and "twitter.com" in url) :
+                    continue
+                google = extract_domain(url)
+                break
+    except Exception: # https://github.com/mxrch/GitFive/issues/15
+        runner.rc.print("[!] Google Search failed, are you using a VPN/Proxy ?", style="italic")
 
     # Hunter.io
     req = httpx.get(f"https://hunter.io/v2/domains-suggestion?query={company}")

gitfive/lib/emails_gen.py

@@ -37,6 +37,17 @@ def generate(runner: GitfiveRunner, custom_domains_list=[], default_domains_list
                 else:
                     usernames.add(sanatize(name.lower()))
 
+    for _, email_data in runner.target.ext_contribs.items():
+        if not (is_local_domain(email_data["domain"]) and email_data["handle"].lower() in config.local_names):
+            usernames.add(email_data["handle"].lower())
+            usernames.add(email_data["handle"].split("+")[0].lower())
+        for name in email_data["names"]:
+            if name and not (is_local_domain(email_data["domain"]) and name.lower() in config.local_names):
+                if " " in name:
+                    fullnames.add(sanatize(name.lower()))
+                else:
+                    usernames.add(sanatize(name.lower()))
+
     for name, name_data in runner.target.near_names.items():
         if name:
             if " " in name:

gitfive/lib/metamon.py

@@ -140,7 +140,7 @@ async def start(runner: GitfiveRunner, emails: List[str]):
     if emails_index:
         # Checking if commits have been pushed
         while True:
-            found, nb_commits = await get_commits_count(runner, f"https://github.com/{runner.creds.username}/{temp_repo_name}")
+            found, nb_commits = await get_commits_count(runner, repo_url=f"https://github.com/{runner.creds.username}/{temp_repo_name}")
             if found and nb_commits == total_commits_count:
                 break
             sleep(1)

gitfive/lib/objects.py

@@ -176,8 +176,8 @@ async def login(self, force=False):
         if req.status_code == 302:
             if req.cookies.get("logged_in") == "yes":
                 self.session = {
-                        "user_session": self._as_client.cookies["user_session"],
-                        "__Host-user_session_same_site": self._as_client.cookies["__Host-user_session_same_site"],
+                        "user_session": req.cookies["user_session"],
+                        "__Host-user_session_same_site": req.cookies["__Host-user_session_same_site"],
                         "_device_id": self._as_client.cookies["_device_id"]
                     }
                 self.save_creds()
@@ -205,8 +205,8 @@ async def login(self, force=False):
                 req = await self._as_client.post("https://github.com/sessions/verified-device", data=data)
                 if req.cookies.get("logged_in") == "yes":
                     self.session = {
-                        "user_session": self._as_client.cookies["user_session"],
-                        "__Host-user_session_same_site": self._as_client.cookies["__Host-user_session_same_site"],
+                        "user_session": req.cookies["user_session"],
+                        "__Host-user_session_same_site": req.cookies["__Host-user_session_same_site"],
                         "_device_id": self._as_client.cookies["_device_id"]
                     }
                     self.save_creds()
@@ -256,8 +256,8 @@ async def login(self, force=False):
                 print("[+] Got confirmation !")
 
                 self.session = {
-                    "user_session": self._as_client.cookies["user_session"],
-                    "__Host-user_session_same_site": self._as_client.cookies["__Host-user_session_same_site"],
+                    "user_session": req.cookies["user_session"],
+                    "__Host-user_session_same_site": req.cookies["__Host-user_session_same_site"],
                     "_device_id": self._as_client.cookies["_device_id"]
                 }
                 self.save_creds()
@@ -284,8 +284,8 @@ async def login(self, force=False):
                 req = await self._as_client.post("https://github.com/sessions/two-factor", data=data)
                 if req.cookies.get("logged_in") == "yes":
                     self.session = {
-                        "user_session": self._as_client.cookies["user_session"],
-                        "__Host-user_session_same_site": self._as_client.cookies["__Host-user_session_same_site"],
+                        "user_session": req.cookies["user_session"],
+                        "__Host-user_session_same_site": req.cookies["__Host-user_session_same_site"],
                         "_device_id": self._as_client.cookies["_device_id"]
                     }
                     self.save_creds()
@@ -347,6 +347,7 @@ def __init__(self):
         self.updated_at: datetime = None
         self.avatar_url = ""
         self.is_default_avatar = True
+        self.nb_ext_contribs = 0
 
         self.potential_friends: Dict[str, Dict[str, int|bool]] = {}
         self.repos: List[Dict[str, any]] = []
@@ -360,6 +361,7 @@ def __init__(self):
         self.ssh_keys: List[str] = []
 
         self.all_contribs: Dict[str, Dict[str, Dict[str, Dict[str, Set[str]]]]] = {}
+        self.ext_contribs: Dict[str, Dict[str, Dict[str, Dict[str, Set[str]]]]] = {}
         self.internal_contribs: Dict[str, Dict[str, Dict[str, Dict[str, Dict[str, Set[str]]]]]] = {"all": {}, "no_github": {}}
         self.usernames_history: Dict[str, Dict[str, Dict[str, Dict[str, Set[str]]]]] = {}
         self.near_names: Dict[str, Dict[str, Dict[str, Dict[str, Dict[str, Dict[str, Set[str]]]]]]] = {}
@@ -433,7 +435,7 @@ def __init__(self):
             "social_follows": trio.CapacityLimiter(50),
             "repos_list": trio.CapacityLimiter(50),
             "commits_scrape": trio.CapacityLimiter(50),
-            "commits_fetch_avatar": trio.CapacityLimiter(4),
+            "commits_fetch_avatar": trio.CapacityLimiter(1), # https://github.com/mxrch/GitFive/issues/3#issuecomment-1321260050
             "orgs_list": trio.CapacityLimiter(50)
         }
 

gitfive/lib/utils.py

@@ -103,9 +103,11 @@ def change_permissions(path: Path|str):
 def show_banner():
     rprint(banner)
 
-async def get_commits_count(runner: GitfiveRunner, repo_url: str):
-    req = await runner.as_client.get(repo_url)
-    body = BeautifulSoup(req.text, 'html.parser')
+async def get_commits_count(runner: GitfiveRunner, repo_url: str="", raw_body: str=""):
+    if not raw_body:
+        req = await runner.as_client.get(repo_url)
+        raw_body = req.text
+    body = BeautifulSoup(raw_body, 'html.parser')
     commits_icon_el = body.find("svg", {"class": "octicon-history"})
     if not commits_icon_el:
         return False, 0
@@ -144,7 +146,12 @@ def humanize_list(array: List[any]):
     return final
 
 def sanatize(text: str) -> str:
-    deaccented = unidecode(text, "utf-8")
+    deaccented = ""
+    try:
+        deaccented = unidecode(text, "utf-8")
+    except Exception:
+        pre_sanatize = ''.join([*filter(lambda x:x.isalpha() or x in "-. ", text)]) # kudos to @n1nj4sec
+        deaccented = unidecode(pre_sanatize, "utf-8")
     return ''.join([*filter(lambda x:x.lower() in string.ascii_lowercase+" ", deaccented)])
 
 def get_gists_stats(runner: GitfiveRunner):
@@ -177,4 +184,11 @@ def unicode_patch(txt: str):
         "ç": "c",
         "à": "a"
     }
-    return txt.replace(''.join([*bad_chars.keys()]), ''.join([*bad_chars.values()]))
+    return txt.replace(''.join([*bad_chars.keys()]), ''.join([*bad_chars.values()]))
+
+def safe_print(txt: str):
+    """
+        Escape the bad characters to avoid ANSI injections.
+        Also works for Rich printers.
+    """
+    return txt.encode("unicode_escape").decode().replace('[', '\\[')