From cb39af39756ac5169a9bdf2722ed2d27a105741e Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Mon, 14 Sep 2026 17:06:18 +0300
Subject: [PATCH 01/13] feat: have get_stats.sh download domains-top-1000.csv
 from s3

---
 get_stats.sh | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/get_stats.sh b/get_stats.sh
index 6536f9b..303f5ca 100755
--- a/get_stats.sh
+++ b/get_stats.sh
@@ -2,6 +2,10 @@
 
 set -o pipefail
 
+TOP_DOMAINS_SOURCE=domains-top-1000-full.csv
+TOP_DOMAINS_TARGET_EXTENSION=domains-top-1000.csv
+TOP_DOMAINS_FOLDER=./stats/top-domains
+
 if aws s3 ls s3://commoncrawl/crawl-analysis/ | sed -E 's@.* @@; s@/$@@' >./stats/crawls.txt; then
     ON_AWS=true;
     echo "Running on AWS (AWS CLI configured for authenticated access)"
@@ -12,15 +16,24 @@ else
     ON_AWS=false
 fi
 
+mkdir -p ${TOP_DOMAINS_FOLDER}
+
 while read crawl; do
     echo $crawl
     if [ -e stats/$crawl.gz ]; then
-        echo "  ... exists"
-        continue
-    fi
-    if $ON_AWS; then
+        echo "  ... stats exist"
+    elif $ON_AWS; then
         aws s3 cp s3://commoncrawl/crawl-analysis/$crawl/stats/part-00000.gz ./stats/$crawl.gz
     else
         curl --silent https://data.commoncrawl.org/crawl-analysis/$crawl/stats/part-00000.gz >./stats/$crawl.gz
     fi
+    # top-domains csv might be missing for older crawls, in that case continue without failing
+    TOP_DOMAINS_TARGET=${TOP_DOMAINS_FOLDER}/${crawl}.${TOP_DOMAINS_TARGET_EXTENSION}
+    if [ -e ${TOP_DOMAINS_TARGET} ]; then
+        echo "  ... top-domains exist"
+    elif $ON_AWS; then
+        aws s3 cp s3://commoncrawl/crawl-analysis/$crawl/stats/${TOP_DOMAINS_SOURCE} ${TOP_DOMAINS_TARGET}
+    else
+        curl --silent --fail https://data.commoncrawl.org/crawl-analysis/$crawl/stats/${TOP_DOMAINS_SOURCE} -o ${TOP_DOMAINS_TARGET} || rm -f ${TOP_DOMAINS_TARGET}
+    fi
 done <./stats/crawls.txt

From 0cef8b298c1344790fac7cd2c02bc1dddc04c645 Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Mon, 14 Sep 2026 18:18:27 +0300
Subject: [PATCH 02/13] feat(domain.py): read domain statistics from downloaded
 s3 file

---
 plot/domain.py | 47 ++++++++++-------------------------------------
 1 file changed, 10 insertions(+), 37 deletions(-)

diff --git a/plot/domain.py b/plot/domain.py
index 7ff64ea..190c280 100644
--- a/plot/domain.py
+++ b/plot/domain.py
@@ -2,50 +2,24 @@
 
 import pandas
 
-from crawlstats import CST, MonthlyCrawl, MultiCount
+from crawlstats import MonthlyCrawl
 from plot.table import TabularStats
 
 
 class DomainStats(TabularStats):
 
-    # defined via crawlstats command-line option --max-top-hosts-domains
-    MAX_TOP_DOMAINS = 500
+    # top domains csv fetched by get_stats.sh
+    MAX_TOP_DOMAINS = 1000
+    TOP_DOMAINS_FILE = 'stats/top-domains/{}.domains-top-{}.csv'
 
     def __init__(self, crawl):
         super().__init__()
         self.crawl = crawl
-        self.N = 0
 
-    def add(self, key, val):
-        cst = CST[key[0]]
-        if cst not in (CST.size, CST.domain):
-            return
-        typeval = key[1]
-        crawl = key[2]
-        if crawl != self.crawl:
-            return
-        if cst == CST.size:
-            self.size[typeval] = val
-            return
-        self.type_stats['domain'][self.N] = typeval 
-        self.type_stats['pages'][self.N] = MultiCount.get_count(0, val)
-        self.type_stats['urls'][self.N] = MultiCount.get_count(1, val)
-        self.type_stats['hosts'][self.N] = MultiCount.get_count(2, val)
-        # self.type_stats['crawl'][self.N] = crawl
-        self.N += 1
-
-    def transform_data(self):
-        data = pandas.DataFrame(self.type_stats)
-        for cnt in ['pages', 'urls']:
-            total = self.size[cnt[:-1]]
-            data['%' + cnt] = 100.0 * data[cnt] / total
-        data.sort_values(ascending=False, inplace=True, by='pages')
-        print(data)
-        self.type_stats = data
-
-    def save_data(self, name, dir_name='data/'):
-        self.type_stats.to_csv('{}/{}-top-{}.csv'.format(self.PLOTDIR, name, self.MAX_TOP_DOMAINS),
-                               float_format='%.6f', index=None)
+    def read_data(self):
+        """Read the downloaded top domains csv."""
+        path = self.TOP_DOMAINS_FILE.format(self.crawl, self.MAX_TOP_DOMAINS)
+        self.type_stats = pandas.read_csv(path)
 
     def plot(self, name):
         data = self.type_stats
@@ -58,6 +32,7 @@ def plot(self, name):
                            float_format='%.6f',
                            classes=css_classes, index='domain'))
 
+
 if __name__ == '__main__':
     plot_crawls = sys.argv[1:]
     if len(plot_crawls) == 0:
@@ -66,7 +41,5 @@ def plot(self, name):
     latest_crawl = plot_crawls[-1]
     plot_name = 'domains'
     plot = DomainStats(latest_crawl)
-    plot.read_from_stdin_or_file()
-    plot.transform_data()
-    plot.save_data(plot_name, dir_name=plot.PLOTDIR)
+    plot.read_data()
     plot.plot(plot_name)

From 2fc1583694d8c9c698c04e7df1217656bbb06e8b Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Mon, 14 Sep 2026 19:07:57 +0300
Subject: [PATCH 03/13] feat(domains.md): increase top-domains count to 1000
 from 500.

---
 plots/domains.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/plots/domains.md b/plots/domains.md
index 9d6af31..02bbaa9 100644
--- a/plots/domains.md
+++ b/plots/domains.md
@@ -1,15 +1,15 @@
 ---
 layout: table
-table_include: domains-top-500.html
+table_include: domains-top-1000.html
 table_sortlist: "{sortList: [[1,1]]}"
 table_searcher: "Filter for domain names"
 ---
 
-Top-500 Registered Domains of the Latest Main Crawl
+Top-1000 Registered Domains of the Latest Main Crawl
 ===================================================
 
-The table below shows the top 500 registered domains (in terms of page captures) of the last main/monthly crawl
-({{ site.latest_crawl }}). The underlying data is also provided in CSV format, see [domains-top-500.csv](./domains-top-500.csv).
+The table below shows the top 1000 registered domains (in terms of page captures) of the last main/monthly crawl
+({{ site.latest_crawl }}). The underlying data is also provided in CSV format, see [domains-top-1000.csv](./domains-top-1000.csv).
 
 Note that the ranking by page captures only partially corresponds to the importance of domains, as the
 crawler respects the robots.txt and tries hard not to overload web servers. Highly ranked domains tend to be

From e73c44931056a3fedc069935c9f84f8c17f3dace Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Mon, 14 Sep 2026 19:08:51 +0300
Subject: [PATCH 04/13] feat(index.md): increase top-domains count to 1000 from
 500.

---
 index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/index.md b/index.md
index 13ddb9d..a80ed84 100644
--- a/index.md
+++ b/index.md
@@ -5,7 +5,7 @@ Statistics of [Common Crawl](https://commoncrawl.org/)'s [web archives](https://
 
 * [size of the crawls](plots/crawlsize) - number of pages, unique URLs, hosts, domains, top-level domains (public suffixes), cumulative growth of crawled data over time
 * [top-level domains](plots/tlds) - distribution and comparison
-* [top-500 registered domains](plots/domains.md)
+* [top-1000 registered domains](plots/domains.md)
 * [crawler-related metrics](plots/crawlermetrics) - fetch status, etc.
 * [overlaps between monthly crawls](plots/crawloverlap)
 * distribution of

From 3c9c1cde542ca11b26cffb0752b1803e168fd104 Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Mon, 14 Sep 2026 19:16:40 +0300
Subject: [PATCH 05/13] fix(domain.py): readd save_data

---
 plot/domain.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/plot/domain.py b/plot/domain.py
index 190c280..6ba98bb 100644
--- a/plot/domain.py
+++ b/plot/domain.py
@@ -21,6 +21,12 @@ def read_data(self):
         path = self.TOP_DOMAINS_FILE.format(self.crawl, self.MAX_TOP_DOMAINS)
         self.type_stats = pandas.read_csv(path)
 
+    def save_data(self, name):
+        """Write the top domains csv next to the html table."""
+        self.type_stats.to_csv('{}/{}-top-{}.csv'.format(
+                                self.PLOTDIR, name, self.MAX_TOP_DOMAINS),
+                               float_format='%.6f', index=None)
+
     def plot(self, name):
         data = self.type_stats
         css_classes = ['tablesorter', 'tablesearcher']
@@ -42,4 +48,5 @@ def plot(self, name):
     plot_name = 'domains'
     plot = DomainStats(latest_crawl)
     plot.read_data()
+    plot.save_data(plot_name)
     plot.plot(plot_name)

From d4bfcfecfc2a48bda4ac9d1d7bc0fbf943956963 Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Thu, 17 Sep 2026 16:52:32 +0300
Subject: [PATCH 06/13] fix(get_stats.sh): update
 domains-top-1000-extended.csv, reflecting change in crawl-tools

---
 get_stats.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/get_stats.sh b/get_stats.sh
index 303f5ca..28d8bd7 100755
--- a/get_stats.sh
+++ b/get_stats.sh
@@ -2,7 +2,7 @@
 
 set -o pipefail
 
-TOP_DOMAINS_SOURCE=domains-top-1000-full.csv
+TOP_DOMAINS_SOURCE=domains-top-1000-extended.csv
 TOP_DOMAINS_TARGET_EXTENSION=domains-top-1000.csv
 TOP_DOMAINS_FOLDER=./stats/top-domains
 

From 61d211facdf2c9a1386bd56d31649fe1b137c1d5 Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Thu, 17 Sep 2026 17:00:39 +0300
Subject: [PATCH 07/13] feat(domain.py,plot.sh): move calculation of
 percentages here from crawl workflow

---
 plot.sh        |  4 ++--
 plot/domain.py | 21 ++++++++++++++++++++-
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/plot.sh b/plot.sh
index 0abf889..274815b 100755
--- a/plot.sh
+++ b/plot.sh
@@ -96,7 +96,7 @@ zcat stats/excerpt/charset.json.gz \
 zcat stats/excerpt/language.json.gz \
     | python3 plot/language.py
 
-zcat stats/excerpt/domain.json.gz \
+zcat stats/excerpt/size.json.gz \
     | python3 plot/domain.py
 
-echo -e "\n\nAll crawl statistics plotted\n"
\ No newline at end of file
+echo -e "\n\nAll crawl statistics plotted\n"
diff --git a/plot/domain.py b/plot/domain.py
index 6ba98bb..c06669b 100644
--- a/plot/domain.py
+++ b/plot/domain.py
@@ -2,7 +2,7 @@
 
 import pandas
 
-from crawlstats import MonthlyCrawl
+from crawlstats import CST, MonthlyCrawl
 from plot.table import TabularStats
 
 
@@ -16,11 +16,28 @@ def __init__(self, crawl):
         super().__init__()
         self.crawl = crawl
 
+    def add(self, key, val):
+        """Collect the crawl size records read from stdin."""
+        cst = CST[key[0]]
+        if cst != CST.size:
+            return
+        if key[2] != self.crawl:
+            return
+        self.size[key[1]] = val
+
     def read_data(self):
         """Read the downloaded top domains csv."""
         path = self.TOP_DOMAINS_FILE.format(self.crawl, self.MAX_TOP_DOMAINS)
         self.type_stats = pandas.read_csv(path)
 
+    def transform_data(self):
+        """Add the percentage columns, counts are crawl totals."""
+        data = self.type_stats
+        for cnt in ['pages', 'urls']:
+            total = self.size[cnt[:-1]]
+            data['%' + cnt] = 100.0 * data[cnt] / total
+        self.type_stats = data
+
     def save_data(self, name):
         """Write the top domains csv next to the html table."""
         self.type_stats.to_csv('{}/{}-top-{}.csv'.format(
@@ -47,6 +64,8 @@ def plot(self, name):
     latest_crawl = plot_crawls[-1]
     plot_name = 'domains'
     plot = DomainStats(latest_crawl)
+    plot.read_from_stdin_or_file()
     plot.read_data()
+    plot.transform_data()
     plot.save_data(plot_name)
     plot.plot(plot_name)

From 157d044138d8dcd5862da31fc8f971970700cadf Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Thu, 17 Sep 2026 17:12:35 +0300
Subject: [PATCH 08/13] fix(domain.py,stats.sh): use 'extended' label for file

---
 get_stats.sh   | 9 ++++-----
 plot/domain.py | 4 ++--
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/get_stats.sh b/get_stats.sh
index 28d8bd7..7985caf 100755
--- a/get_stats.sh
+++ b/get_stats.sh
@@ -2,8 +2,7 @@
 
 set -o pipefail
 
-TOP_DOMAINS_SOURCE=domains-top-1000-extended.csv
-TOP_DOMAINS_TARGET_EXTENSION=domains-top-1000.csv
+TOP_DOMAINS_FILE=domains-top-1000-extended.csv
 TOP_DOMAINS_FOLDER=./stats/top-domains
 
 if aws s3 ls s3://commoncrawl/crawl-analysis/ | sed -E 's@.* @@; s@/$@@' >./stats/crawls.txt; then
@@ -28,12 +27,12 @@ while read crawl; do
         curl --silent https://data.commoncrawl.org/crawl-analysis/$crawl/stats/part-00000.gz >./stats/$crawl.gz
     fi
     # top-domains csv might be missing for older crawls, in that case continue without failing
-    TOP_DOMAINS_TARGET=${TOP_DOMAINS_FOLDER}/${crawl}.${TOP_DOMAINS_TARGET_EXTENSION}
+    TOP_DOMAINS_TARGET=${TOP_DOMAINS_FOLDER}/${crawl}.${TOP_DOMAINS_FILE}
     if [ -e ${TOP_DOMAINS_TARGET} ]; then
         echo "  ... top-domains exist"
     elif $ON_AWS; then
-        aws s3 cp s3://commoncrawl/crawl-analysis/$crawl/stats/${TOP_DOMAINS_SOURCE} ${TOP_DOMAINS_TARGET}
+        aws s3 cp s3://commoncrawl/crawl-analysis/$crawl/stats/${TOP_DOMAINS_FILE} ${TOP_DOMAINS_TARGET}
     else
-        curl --silent --fail https://data.commoncrawl.org/crawl-analysis/$crawl/stats/${TOP_DOMAINS_SOURCE} -o ${TOP_DOMAINS_TARGET} || rm -f ${TOP_DOMAINS_TARGET}
+        curl --silent --fail https://data.commoncrawl.org/crawl-analysis/$crawl/stats/${TOP_DOMAINS_FILE} -o ${TOP_DOMAINS_TARGET} || rm -f ${TOP_DOMAINS_TARGET}
     fi
 done <./stats/crawls.txt
diff --git a/plot/domain.py b/plot/domain.py
index c06669b..4147dc4 100644
--- a/plot/domain.py
+++ b/plot/domain.py
@@ -8,9 +8,9 @@
 
 class DomainStats(TabularStats):
 
-    # top domains csv fetched by get_stats.sh
+    # extended top domains csv fetched by get_stats.sh
     MAX_TOP_DOMAINS = 1000
-    TOP_DOMAINS_FILE = 'stats/top-domains/{}.domains-top-{}.csv'
+    TOP_DOMAINS_FILE = 'stats/top-domains/{}.domains-top-{}-extended.csv'
 
     def __init__(self, crawl):
         super().__init__()

From 9551a3baf7806fbb48729e034649d1f1cd82e86d Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Fri, 18 Sep 2026 15:59:57 +0300
Subject: [PATCH 09/13] feat(get_stats.sh): streamline s3 paths so they can be
 overriden for test

---
 get_stats.sh | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/get_stats.sh b/get_stats.sh
index 7985caf..8dc318d 100755
--- a/get_stats.sh
+++ b/get_stats.sh
@@ -1,15 +1,29 @@
 #!/bin/bash
 
+# Download crawl statistics and top-domains csv files into ./stats/
+#
+# Usage: ./get_stats.sh
+# Test: To test top-domains feature, override TOP_DOMAINS_S3/URL, eg.
+#           TOP_DOMAINS_S3=s3://<test-bucket>/<test-prefix>/crawl-analysis ./get_stats.sh
+
 set -o pipefail
 
 TOP_DOMAINS_FILE=domains-top-1000-extended.csv
 TOP_DOMAINS_FOLDER=./stats/top-domains
 
-if aws s3 ls s3://commoncrawl/crawl-analysis/ | sed -E 's@.* @@; s@/$@@' >./stats/crawls.txt; then
+CRAWL_ANALYSIS_S3=s3://commoncrawl/crawl-analysis
+CRAWL_ANALYSIS_URL=https://data.commoncrawl.org/crawl-analysis
+
+# Override to test, see above.
+TOP_DOMAINS_S3=${TOP_DOMAINS_S3:-$CRAWL_ANALYSIS_S3}
+TOP_DOMAINS_URL=${TOP_DOMAINS_URL:-$CRAWL_ANALYSIS_URL}
+
+
+if aws s3 ls ${CRAWL_ANALYSIS_S3}/ | sed -E 's@.* @@; s@/$@@' >./stats/crawls.txt; then
     ON_AWS=true;
     echo "Running on AWS (AWS CLI configured for authenticated access)"
 else
-    echo "Downloading from https://data.commoncrawl.org/ using curl"
+    echo "Downloading from ${CRAWL_ANALYSIS_URL} using curl"
     # list of crawls enumerated in crawlstats.py
     python3 -c 'from crawlstats import MonthlyCrawl; [print(c) for c in sorted(MonthlyCrawl.by_name.keys())]' >./stats/crawls.txt
     ON_AWS=false
@@ -22,17 +36,17 @@ while read crawl; do
     if [ -e stats/$crawl.gz ]; then
         echo "  ... stats exist"
     elif $ON_AWS; then
-        aws s3 cp s3://commoncrawl/crawl-analysis/$crawl/stats/part-00000.gz ./stats/$crawl.gz
+        aws s3 cp ${CRAWL_ANALYSIS_S3}/$crawl/stats/part-00000.gz ./stats/$crawl.gz
     else
-        curl --silent https://data.commoncrawl.org/crawl-analysis/$crawl/stats/part-00000.gz >./stats/$crawl.gz
+        curl --silent ${CRAWL_ANALYSIS_URL}/$crawl/stats/part-00000.gz >./stats/$crawl.gz
     fi
     # top-domains csv might be missing for older crawls, in that case continue without failing
     TOP_DOMAINS_TARGET=${TOP_DOMAINS_FOLDER}/${crawl}.${TOP_DOMAINS_FILE}
     if [ -e ${TOP_DOMAINS_TARGET} ]; then
         echo "  ... top-domains exist"
     elif $ON_AWS; then
-        aws s3 cp s3://commoncrawl/crawl-analysis/$crawl/stats/${TOP_DOMAINS_FILE} ${TOP_DOMAINS_TARGET}
+        aws s3 cp ${TOP_DOMAINS_S3}/$crawl/stats/${TOP_DOMAINS_FILE} ${TOP_DOMAINS_TARGET}
     else
-        curl --silent --fail https://data.commoncrawl.org/crawl-analysis/$crawl/stats/${TOP_DOMAINS_FILE} -o ${TOP_DOMAINS_TARGET} || rm -f ${TOP_DOMAINS_TARGET}
+        curl --silent --fail ${TOP_DOMAINS_URL}/$crawl/stats/${TOP_DOMAINS_FILE} -o ${TOP_DOMAINS_TARGET} || rm -f ${TOP_DOMAINS_TARGET}
     fi
 done <./stats/crawls.txt

From 5836eace41a6a3ab29469134123bb5ffc3e2cab9 Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Fri, 18 Sep 2026 16:07:27 +0300
Subject: [PATCH 10/13] feat(get_stats.sh): make curl branch fail on unexisting
 top-domains files

---
 get_stats.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/get_stats.sh b/get_stats.sh
index 8dc318d..56bbce6 100755
--- a/get_stats.sh
+++ b/get_stats.sh
@@ -47,6 +47,6 @@ while read crawl; do
     elif $ON_AWS; then
         aws s3 cp ${TOP_DOMAINS_S3}/$crawl/stats/${TOP_DOMAINS_FILE} ${TOP_DOMAINS_TARGET}
     else
-        curl --silent --fail ${TOP_DOMAINS_URL}/$crawl/stats/${TOP_DOMAINS_FILE} -o ${TOP_DOMAINS_TARGET} || rm -f ${TOP_DOMAINS_TARGET}
+        curl --silent --fail ${TOP_DOMAINS_URL}/$crawl/stats/${TOP_DOMAINS_FILE} -o ${TOP_DOMAINS_TARGET} || { rm -f ${TOP_DOMAINS_TARGET}; false; }
     fi
 done <./stats/crawls.txt

From e97a640f150435a5b72531bf30b557585a94e9be Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Fri, 18 Sep 2026 17:06:44 +0300
Subject: [PATCH 11/13] fix(domain.py): correct name collusion read_data ->
 read_top_domains

---
 plot/domain.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plot/domain.py b/plot/domain.py
index 4147dc4..c1b9a89 100644
--- a/plot/domain.py
+++ b/plot/domain.py
@@ -25,7 +25,7 @@ def add(self, key, val):
             return
         self.size[key[1]] = val
 
-    def read_data(self):
+    def read_top_domains(self):
         """Read the downloaded top domains csv."""
         path = self.TOP_DOMAINS_FILE.format(self.crawl, self.MAX_TOP_DOMAINS)
         self.type_stats = pandas.read_csv(path)
@@ -65,7 +65,7 @@ def plot(self, name):
     plot_name = 'domains'
     plot = DomainStats(latest_crawl)
     plot.read_from_stdin_or_file()
-    plot.read_data()
+    plot.read_top_domains()
     plot.transform_data()
     plot.save_data(plot_name)
     plot.plot(plot_name)

From 7521b9f1eab317a65d065382be6f023ba2af901d Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Thu, 24 Sep 2026 19:07:24 +0300
Subject: [PATCH 12/13] feat(get_stats.sh): download gzipped file. update
 bucket name placeholder.

---
 get_stats.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/get_stats.sh b/get_stats.sh
index 56bbce6..bd6ed7b 100755
--- a/get_stats.sh
+++ b/get_stats.sh
@@ -3,12 +3,13 @@
 # Download crawl statistics and top-domains csv files into ./stats/
 #
 # Usage: ./get_stats.sh
-# Test: To test top-domains feature, override TOP_DOMAINS_S3/URL, eg.
-#           TOP_DOMAINS_S3=s3://<test-bucket>/<test-prefix>/crawl-analysis ./get_stats.sh
+# Test:  Override TOP_DOMAINS_S3/URL, eg.
+#        TOP_DOMAINS_S3=s3://test-bucket/test-prefix/crawl-analysis ./get_stats.sh
+
 
 set -o pipefail
 
-TOP_DOMAINS_FILE=domains-top-1000-extended.csv
+TOP_DOMAINS_FILE=domains-top-1000-extended.csv.gz
 TOP_DOMAINS_FOLDER=./stats/top-domains
 
 CRAWL_ANALYSIS_S3=s3://commoncrawl/crawl-analysis

From 0fbc848c512f7ebea38ba1814a1e53d485ce1c9c Mon Sep 17 00:00:00 2001
From: Hande Celikkanat <7702228+handecelikkanat@users.noreply.github.com>
Date: Thu, 24 Sep 2026 19:16:56 +0300
Subject: [PATCH 13/13] feat(domain.py): add gzip handling

---
 plot/domain.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/plot/domain.py b/plot/domain.py
index c1b9a89..13e7a86 100644
--- a/plot/domain.py
+++ b/plot/domain.py
@@ -10,7 +10,7 @@ class DomainStats(TabularStats):
 
     # extended top domains csv fetched by get_stats.sh
     MAX_TOP_DOMAINS = 1000
-    TOP_DOMAINS_FILE = 'stats/top-domains/{}.domains-top-{}-extended.csv'
+    TOP_DOMAINS_FILE = 'stats/top-domains/{}.domains-top-{}-extended.csv.gz'
 
     def __init__(self, crawl):
         super().__init__()
@@ -26,9 +26,11 @@ def add(self, key, val):
         self.size[key[1]] = val
 
     def read_top_domains(self):
-        """Read the downloaded top domains csv."""
+        """Read the downloaded top domains csv, handles gzipped or plain versions."""
         path = self.TOP_DOMAINS_FILE.format(self.crawl, self.MAX_TOP_DOMAINS)
-        self.type_stats = pandas.read_csv(path)
+        with open(path, 'rb') as instream:
+            gzipped = instream.read(2) == b'\x1f\x8b'
+        self.type_stats = pandas.read_csv(path, compression='gzip' if gzipped else None)
 
     def transform_data(self):
         """Add the percentage columns, counts are crawl totals."""