Collect existing datapackages

[davidgasquez]

Basically, get these files: https://github.com/search?q=path%3A**%2Fdatapackage.json&type=code

[davidgasquez]

Some quick code to do that. 👇

import requests

def search_code(query, page=0):
    url = "https://api.github.com/search/code"
    headers = {
        "Accept": "application/vnd.github+json",
        "Authorization": f"Bearer {os.getenv("BEARER")}"
    }

    # Define the query parameters
    params = {
        "q": query,
        "page": page
    }

    # Make the GET request
    response = requests.get(url, headers=headers, params=params)
    response.raise_for_status()
    return response.json()

data = search_code("filename:datapackage.yaml")

all_items = []
page = 1

while page:
    items = search_code("filename:datapackage.yaml", page)['items']
    all_items.extend(items)
    if not items:
        page = None
    else:
        page = page + 1