gowarc/protocol.go at main · saveweb/gowarc · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
package warc

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"
	"net"
	"os"

	http "github.com/saveweb/fhttp"
)

type protocolInfo struct {
	Protocol    string
	TLSVersion  uint16
	CipherSuite uint16
	QUICVersion uint32
	RemoteAddr  net.Addr
}

type unknownProtocol struct {
	Layer string
	Value string
}

const logDefaultFlags = log.LstdFlags

var protocolWarningLogger = log.New(os.Stderr, "gowarc: ", logDefaultFlags)

var knownWARCProtocolIDs = map[string]struct{}{
	"dns": {}, "ftp": {}, "gemini": {}, "gopher": {},
	"http/0.9": {}, "http/1.0": {}, "http/1.1": {}, "h2": {}, "h2c": {}, "h3": {},
	"quic/1": {}, "quic/2": {},
	"spdy/1": {}, "spdy/2": {}, "spdy/3": {},
	"ssl/2": {}, "ssl/3": {},
	"tls/1.0": {}, "tls/1.1": {}, "tls/1.2": {}, "tls/1.3": {},
}

const (
	quicVersion1 uint32 = 0x00000001
	quicVersion2 uint32 = 0x6b3343cf
)

// WARCProtocolValues maps transport metadata to the identifiers registered by
// the IIPC WARC-Protocol proposal. Unknown non-empty layers are returned for
// warning and deliberately omitted from the WARC record.
func (pi *protocolInfo) WARCProtocolValues(applicationProtocol string) ([]string, []unknownProtocol) {
	var values []string
	var unknown []unknownProtocol
	add := func(layer, value string) {
		if value == "" {
			return
		}
		if _, ok := knownWARCProtocolIDs[value]; !ok {
			unknown = append(unknown, unknownProtocol{Layer: layer, Value: value})
			return
		}
		values = append(values, value)
	}

	add("application", applicationProtocol)
	switch pi.QUICVersion {
	case 0:
	case quicVersion1:
		add("transport", "quic/1")
	case quicVersion2:
		add("transport", "quic/2")
	default:
		unknown = append(unknown, unknownProtocol{Layer: "transport", Value: fmt.Sprintf("quic version %#x", pi.QUICVersion)})
	}
	switch pi.TLSVersion {
	case 0:
	case tls.VersionTLS10:
		add("security", "tls/1.0")
	case tls.VersionTLS11:
		add("security", "tls/1.1")
	case tls.VersionTLS12:
		add("security", "tls/1.2")
	case tls.VersionTLS13:
		add("security", "tls/1.3")
	default:
		unknown = append(unknown, unknownProtocol{Layer: "security", Value: fmt.Sprintf("TLS version %#x", pi.TLSVersion)})
	}
	return values, unknown
}

func warnUnknownProtocol(value unknownProtocol) {
	protocolWarningLogger.Printf("WARNING: omitting unknown WARC-Protocol layer=%s value=%q", value.Layer, value.Value)
}

func addWARCProtocols(header Header, pi *protocolInfo, applicationProtocol string, warned map[unknownProtocol]struct{}) {
	protocols, unknownProtocols := pi.WARCProtocolValues(applicationProtocol)
	for _, value := range protocols {
		header.Add("WARC-Protocol", value)
	}
	for _, value := range unknownProtocols {
		if _, alreadyWarned := warned[value]; alreadyWarned {
			continue
		}
		warned[value] = struct{}{}
		warnUnknownProtocol(value)
	}
}

func tlsCipherSuiteName(cs uint16) string {
	return tls.CipherSuiteName(cs)
}

type protocolClient interface {
	Do(ctx context.Context, req *http.Request) (*http.Response, error)
	CloseIdleConnections()
	Shutdown()
}