236 lines
6.2 KiB
Python
236 lines
6.2 KiB
Python
#!/usr/bin/env python
|
|
|
|
"""Collect data on the most-starred repos using GitHub's GraphQL API."""
|
|
|
|
import json
|
|
import os
|
|
import requests
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
|
|
from config import Parser
|
|
|
|
token = Parser.GithubToken
|
|
|
|
def query(payload, variables=None):
|
|
r = requests.post(
|
|
'https://api.github.com/graphql',
|
|
headers={'Authorization': f'bearer {token}'},
|
|
json={"query": payload, "variables": variables or {}}
|
|
)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
|
|
repo_query = '''
|
|
query popular_repos($start: String, $num: Int!){
|
|
rateLimit {
|
|
cost
|
|
remaining
|
|
resetAt
|
|
}
|
|
search(query: "is:public %s", type: REPOSITORY, first: $num, after: $start) {
|
|
repositoryCount
|
|
pageInfo {
|
|
hasNextPage
|
|
endCursor
|
|
}
|
|
edges {
|
|
node {
|
|
... on Repository {
|
|
nameWithOwner
|
|
createdAt
|
|
forkCount
|
|
isFork
|
|
updatedAt
|
|
primaryLanguage {
|
|
name
|
|
}
|
|
stargazers {
|
|
totalCount
|
|
}
|
|
watchers {
|
|
totalCount
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
'''
|
|
|
|
count_query = '''
|
|
query {
|
|
rateLimit {
|
|
cost
|
|
remaining
|
|
resetAt
|
|
}
|
|
search(query: "is:public %s", type: REPOSITORY, first: 1) {
|
|
repositoryCount
|
|
}
|
|
}
|
|
'''
|
|
|
|
def get_repos(q, cursor, num):
|
|
return query(repo_query % q, {'start': cursor, 'num': num})['data']
|
|
|
|
|
|
def get_count(q):
|
|
return query(count_query % q)['data']['search']['repositoryCount']
|
|
|
|
|
|
def scrape(q, out_file):
|
|
path = f'responses/{out_file}'
|
|
if os.path.exists(path):
|
|
print('Skipping', path, 'already exists')
|
|
return
|
|
all_repos = []
|
|
cursor = None
|
|
print('Creating', path)
|
|
while True:
|
|
r = get_repos(q, cursor, 100)
|
|
search = r['search']
|
|
pi = search['pageInfo']
|
|
cursor = pi['endCursor']
|
|
has_next = pi['hasNextPage']
|
|
total = search['repositoryCount']
|
|
if total > 2000:
|
|
raise ValueError(f'Too many results for {q}: {total}')
|
|
all_repos += [e['node'] for e in search['edges']]
|
|
print(r['rateLimit'])
|
|
print(len(all_repos), ' / ', total, cursor)
|
|
if not has_next or r['rateLimit']['remaining'] < 10:
|
|
break
|
|
#print(all_repos)
|
|
from pymongo import MongoClient
|
|
client = MongoClient("mongodb://admin:admin@localhost:27017")
|
|
db = client['git']
|
|
collection = db['repos']
|
|
for repo in all_repos:
|
|
entity = {}
|
|
filter_dict = {}
|
|
|
|
for key, value in repo.items():
|
|
if key == "nameWithOwner":
|
|
filter_dict[key] = value
|
|
else:
|
|
entity[key] = value
|
|
|
|
collection.update_one(filter_dict, {"$set": entity}, upsert=True)
|
|
|
|
with open(path, 'w') as out:
|
|
print(out)
|
|
#json.dump(all_repos, out)
|
|
time.sleep(4)
|
|
|
|
|
|
def scrape_star_range(low, high):
|
|
"""Scrape a simple star range [low, high]."""
|
|
out_file = f'repos.stars={low}..{high}.json'
|
|
q = 'stars:%d..%d' % (low, high)
|
|
scrape(q, out_file)
|
|
|
|
|
|
def scrape_breaks():
|
|
breaks = json.load(open('breaks.json'))
|
|
for hi, lo in zip(breaks[:-1], breaks[1:]):
|
|
scrape_star_range(lo, hi - 1)
|
|
|
|
|
|
def scrape_star_dates():
|
|
for stars in range(123, 15, -1):
|
|
out_file = f'repos.star={stars}.-2015.json'
|
|
q = 'stars:%d created:<=2015' % stars
|
|
scrape(q, out_file)
|
|
|
|
out_file = f'repos.star={stars}.2016-.json'
|
|
q = 'stars:%d created:>=2016' % stars
|
|
scrape(q, out_file)
|
|
|
|
|
|
def query_for_star_years(stars, start, end):
|
|
q = 'stars:%s' % stars
|
|
if start == 2010 and end == 2023:
|
|
return q
|
|
elif start == 2010:
|
|
return f'{q} created:<={end}'
|
|
elif end == 2023:
|
|
return f'{q} created:>={start}'
|
|
else:
|
|
return f'{q} created:{start}..{end}'
|
|
|
|
|
|
def split_interval(a, b):
|
|
d = int((b - a) / 2)
|
|
return [(a, a + d), (a + d + 1, b)]
|
|
|
|
|
|
def split_by_year(stars, start, end):
|
|
if start == 2010 and end == 2023:
|
|
c = 1001 # we know this will fail.
|
|
elif start == end:
|
|
split_by_days(
|
|
stars,
|
|
datetime(start, 1, 1),
|
|
datetime(start, 12, 31)
|
|
)
|
|
return
|
|
else:
|
|
q = query_for_star_years(stars, start, end)
|
|
c = get_count(q)
|
|
if c <= 1000:
|
|
out_file = f'repos.star={stars}.{start}-{end}.json'
|
|
print(f'query: {q}')
|
|
scrape(q, out_file)
|
|
else:
|
|
if start == end:
|
|
raise ValueError(f'Can\'t split any more for {stars} / {start}')
|
|
print(f'{stars} {start}..{end} -> {c}, will split')
|
|
for a, b in split_interval(start, end):
|
|
split_by_year(stars, a, b)
|
|
|
|
|
|
def split_by_days(stars, day_start, day_end):
|
|
start_fmt = day_start.strftime('%Y-%m-%d')
|
|
end_fmt = day_end.strftime('%Y-%m-%d')
|
|
q = query_for_star_years(stars, start_fmt, end_fmt)
|
|
c = get_count(q)
|
|
if c <= 1000:
|
|
out_file = f'repos.star={stars}.{start_fmt}-{end_fmt}.json'
|
|
print(f'query: {q}')
|
|
scrape(q, out_file)
|
|
else:
|
|
days = (day_end - day_start).days
|
|
if days == 0:
|
|
raise ValueError(f'Can\'t split any more: {stars} / {day_start} .. {day_end}')
|
|
for a, b in split_interval(0, days):
|
|
dt_a = day_start + timedelta(days=a)
|
|
dt_b = day_start + timedelta(days=b)
|
|
split_by_days(stars, dt_a, dt_b)
|
|
|
|
|
|
def scrape_star_dates_split():
|
|
#for stars in range(83, 15, -1):
|
|
for stars in range(40, 15, -1):
|
|
split_by_year(stars, 2010, 2023)
|
|
|
|
|
|
def scrape_range_days():
|
|
# Scrape from a low star range up, splitting by creation date (which never changes).
|
|
# ranges = [(15, 20), (21, 25), (26, 30), (31, 35), (36, 40), (41, 45), (46, 49)]
|
|
#ranges = [(50, 60), (61, 70), (71, 80), (81, 90), (91, 100)]
|
|
#ranges = ranges + [(100, 119), (120, 139), (140, 159), (160, 179), (180, 200)]
|
|
#ranges = ranges + [(201, 225), (226, 250), (251, 300), (301, 400), (401, 500)]
|
|
#ranges = ranges + [(501, 700), (701, 1000), (1001, 1500), (1501, 5000), (5001, 1_000_000)]
|
|
ranges = [(1001, 1500), (1501, 5000), (5001, 1_000_000)]
|
|
for a, b in ranges:
|
|
stars = f'{a}..{b}'
|
|
split_by_days(stars, datetime(2007, 1, 1), datetime(2024, 2, 2))
|
|
|
|
|
|
#if __name__ == '__main__':
|
|
# scrape_breaks()
|
|
# scrape_star_dates()
|
|
# scrape_star_dates_split()
|
|
#scrape_range_days()
|