count_responses.sh

#!/bin/bash
# Saves total response count to responses folder
# Returns total byte count to size folder
# Naive testing showed wc --bytes faster than dd or pv (with pv not being 100% accurate)
# Example usage: shuf ~/warc_list.txt | parallel --no-notice --bar -P 64 ./count_responses.sh {}
#
mkdir -p responses
mkdir -p size
###
# Get the data from S3 and send it to stdout, where zcat decompresses it
s3cmd get s3://commoncrawl/$1 - | zcat | \
# send the data two places, first to count how many bytes and record that
tee >(wc --bytes > size/`basename $1`) | \
# second to find all response results and count the number of times that's seen
grep "WARC-Type: response" | wc -l > responses/`basename $1`
###
# Helpful:
# As there's no good error checking, errors sometimes pop up leaving empty or tiny results
# cat * | grep -E '[0-9]{6,10}' | wc -l
# cat * | grep -E '[0-9]{6,10}' | awk '{ SUM += $1} END { print SUM }'
###
# cat * | grep -E '[0-9]{4,10}' | wc -l
# cat * | grep -E '[0-9]{4,10}' | awk '{ SUM += $1} END { print SUM }'