sources.go 7.68 KB
Newer Older
1 2 3
package main

import (
Frank Denis's avatar
Frank Denis committed
4
	"errors"
5
	"fmt"
6
	"io"
7 8
	"io/ioutil"
	"net/http"
9
	"net/url"
10
	"os"
11
	"path/filepath"
12 13
	"strings"
	"time"
14
	"unicode"
15 16 17 18

	"github.com/dchest/safefile"

	"github.com/jedisct1/dlog"
Frank Denis's avatar
Frank Denis committed
19
	stamps "github.com/jedisct1/go-dnsstamps"
20 21 22 23 24 25
	"github.com/jedisct1/go-minisign"
)

type SourceFormat int

const (
26
	SourceFormatV2 = iota
27 28
)

29
const (
Frank Denis's avatar
Frank Denis committed
30
	SourcesUpdateDelay = time.Duration(24) * time.Hour
31 32
)

33
type Source struct {
34
	urls   []string
35 36 37 38
	format SourceFormat
	in     string
}

39 40
func fetchFromCache(cacheFile string) (in string, expired bool, delayTillNextUpdate time.Duration, err error) {
	expired = false
Frank Denis's avatar
Frank Denis committed
41 42
	fi, err := os.Stat(cacheFile)
	if err != nil {
43
		dlog.Debugf("Cache file [%s] not present", cacheFile)
Frank Denis's avatar
Frank Denis committed
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
		delayTillNextUpdate = time.Duration(0)
		return
	}
	elapsed := time.Since(fi.ModTime())
	if elapsed < SourcesUpdateDelay {
		dlog.Debugf("Cache file [%s] is still fresh", cacheFile)
		delayTillNextUpdate = SourcesUpdateDelay - elapsed
	} else {
		dlog.Debugf("Cache file [%s] needs to be refreshed", cacheFile)
		delayTillNextUpdate = time.Duration(0)
	}
	var bin []byte
	bin, err = ioutil.ReadFile(cacheFile)
	if err != nil {
		delayTillNextUpdate = time.Duration(0)
		return
	}
	in = string(bin)
62 63 64
	if delayTillNextUpdate <= time.Duration(0) {
		expired = true
	}
Frank Denis's avatar
Frank Denis committed
65
	return
66 67
}

68
func fetchWithCache(xTransport *XTransport, urlStr string, cacheFile string) (in string, cached bool, delayTillNextUpdate time.Duration, err error) {
Frank Denis's avatar
Frank Denis committed
69
	cached = false
70 71 72
	expired := false
	in, expired, delayTillNextUpdate, err = fetchFromCache(cacheFile)
	if err == nil && !expired {
Frank Denis's avatar
Frank Denis committed
73 74 75 76
		dlog.Debugf("Delay till next update: %v", delayTillNextUpdate)
		cached = true
		return
	}
77 78 79
	if expired {
		cached = true
	}
80
	if len(urlStr) == 0 {
81 82 83
		if !expired {
			err = fmt.Errorf("Cache file [%s] not present and no URL given to retrieve it", cacheFile)
		}
84 85 86
		return
	}

Frank Denis's avatar
Frank Denis committed
87
	var resp *http.Response
88 89 90 91 92 93
	dlog.Infof("Loading source information from URL [%s]", urlStr)

	url, err := url.Parse(urlStr)
	if err != nil {
		return
	}
94
	resp, _, err = xTransport.Get(url, "", 30*time.Second)
Frank Denis's avatar
Frank Denis committed
95 96 97 98 99 100 101 102
	if err == nil && resp != nil && (resp.StatusCode < 200 || resp.StatusCode > 299) {
		err = fmt.Errorf("Webserver returned code %d", resp.StatusCode)
		return
	} else if err != nil {
		return
	} else if resp == nil {
		err = errors.New("Webserver returned an error")
		return
103
	}
Frank Denis's avatar
Frank Denis committed
104
	var bin []byte
105
	bin, err = ioutil.ReadAll(io.LimitReader(resp.Body, MaxHTTPBodyLength))
Frank Denis's avatar
Frank Denis committed
106 107 108
	resp.Body.Close()
	if err != nil {
		return
109
	}
Frank Denis's avatar
Frank Denis committed
110
	err = nil
111
	cached = false
112
	in = string(bin)
Frank Denis's avatar
Frank Denis committed
113
	delayTillNextUpdate = SourcesUpdateDelay
114 115 116 117 118 119 120
	return
}

func AtomicFileWrite(file string, data []byte) error {
	return safefile.WriteFile(file, data, 0644)
}

121 122 123 124 125 126
type URLToPrefetch struct {
	url       string
	cacheFile string
	when      time.Time
}

127
func NewSource(xTransport *XTransport, urls []string, minisignKeyStr string, cacheFile string, formatStr string, refreshDelay time.Duration) (Source, []URLToPrefetch, error) {
Frank Denis's avatar
Frank Denis committed
128
	_ = refreshDelay
129
	source := Source{urls: urls}
130
	if formatStr == "v2" {
131 132
		source.format = SourceFormatV2
	} else {
133
		return source, []URLToPrefetch{}, fmt.Errorf("Unsupported source format: [%s]", formatStr)
134 135 136
	}
	minisignKey, err := minisign.NewPublicKey(minisignKeyStr)
	if err != nil {
137 138
		return source, []URLToPrefetch{}, err
	}
Frank Denis's avatar
Frank Denis committed
139 140
	now := time.Now()
	urlsToPrefetch := []URLToPrefetch{}
141
	sigCacheFile := cacheFile + ".minisig"
Frank Denis's avatar
Frank Denis committed
142

143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
	var sigStr, in string
	var cached, sigCached bool
	var delayTillNextUpdate, sigDelayTillNextUpdate time.Duration
	var sigErr error
	var preloadURL string
	if len(urls) <= 0 {
		in, cached, delayTillNextUpdate, err = fetchWithCache(xTransport, "", cacheFile)
		sigStr, sigCached, sigDelayTillNextUpdate, sigErr = fetchWithCache(xTransport, "", sigCacheFile)
	} else {
		preloadURL = urls[0]
		for _, url := range urls {
			sigURL := url + ".minisig"
			in, cached, delayTillNextUpdate, err = fetchWithCache(xTransport, url, cacheFile)
			sigStr, sigCached, sigDelayTillNextUpdate, sigErr = fetchWithCache(xTransport, sigURL, sigCacheFile)
			if err == nil && sigErr == nil {
				preloadURL = url
				break
			}
			dlog.Infof("Loading from [%s] failed", url)
162
		}
163 164 165 166 167 168 169 170 171 172 173
	}
	if len(preloadURL) > 0 {
		url := preloadURL
		sigURL := url + ".minisig"
		urlsToPrefetch = append(urlsToPrefetch, URLToPrefetch{url: url, cacheFile: cacheFile, when: now.Add(delayTillNextUpdate)})
		urlsToPrefetch = append(urlsToPrefetch, URLToPrefetch{url: sigURL, cacheFile: sigCacheFile, when: now.Add(sigDelayTillNextUpdate)})
	}
	if sigErr != nil && err == nil {
		err = sigErr
	}
	if err != nil {
174
		return source, urlsToPrefetch, err
Frank Denis's avatar
Frank Denis committed
175 176
	}

177 178
	signature, err := minisign.DecodeSignature(sigStr)
	if err != nil {
179 180
		os.Remove(cacheFile)
		os.Remove(sigCacheFile)
181
		return source, urlsToPrefetch, err
182 183
	}
	res, err := minisignKey.Verify([]byte(in), signature)
Frank Denis's avatar
Frank Denis committed
184
	if err != nil || !res {
185 186
		os.Remove(cacheFile)
		os.Remove(sigCacheFile)
187
		return source, urlsToPrefetch, err
188
	}
Frank Denis's avatar
Frank Denis committed
189
	if !cached {
190
		if err = AtomicFileWrite(cacheFile, []byte(in)); err != nil {
191 192 193
			if absPath, err2 := filepath.Abs(cacheFile); err2 == nil {
				dlog.Warnf("%s: %s", absPath, err)
			}
194 195
		}
	}
Frank Denis's avatar
Frank Denis committed
196
	if !sigCached {
197
		if err = AtomicFileWrite(sigCacheFile, []byte(sigStr)); err != nil {
198 199 200
			if absPath, err2 := filepath.Abs(sigCacheFile); err2 == nil {
				dlog.Warnf("%s: %s", absPath, err)
			}
201 202
		}
	}
203
	dlog.Noticef("Source [%s] loaded", cacheFile)
204
	source.in = in
205
	return source, urlsToPrefetch, nil
206 207
}

208
func (source *Source) Parse(prefix string) ([]RegisteredServer, error) {
209
	if source.format == SourceFormatV2 {
210 211 212 213 214 215 216 217 218 219 220
		return source.parseV2(prefix)
	}
	dlog.Fatal("Unexpected source format")
	return []RegisteredServer{}, nil
}

func (source *Source) parseV2(prefix string) ([]RegisteredServer, error) {
	var registeredServers []RegisteredServer
	in := string(source.in)
	parts := strings.Split(in, "## ")
	if len(parts) < 2 {
221
		return registeredServers, fmt.Errorf("Invalid format for source at [%v]", source.urls)
222 223 224 225 226 227
	}
	parts = parts[1:]
	for _, part := range parts {
		part = strings.TrimFunc(part, unicode.IsSpace)
		subparts := strings.Split(part, "\n")
		if len(subparts) < 2 {
228
			return registeredServers, fmt.Errorf("Invalid format for source at [%v]", source.urls)
229 230 231
		}
		name := strings.TrimFunc(subparts[0], unicode.IsSpace)
		if len(name) == 0 {
232
			return registeredServers, fmt.Errorf("Invalid format for source at [%v]", source.urls)
233
		}
234
		subparts = subparts[1:]
235
		name = prefix + name
236
		var stampStr, description string
237 238 239
		for _, subpart := range subparts {
			subpart = strings.TrimFunc(subpart, unicode.IsSpace)
			if strings.HasPrefix(subpart, "sdns://") {
240
				if len(stampStr) > 0 {
241
					return registeredServers, fmt.Errorf("Multiple stamps for server [%s] in source from [%v]", name, source.urls)
242
				}
243
				stampStr = subpart
244 245 246
				continue
			} else if len(subpart) == 0 || strings.HasPrefix(subpart, "//") {
				continue
247
			}
248 249 250 251
			if len(description) > 0 {
				description += "\n"
			}
			description += subpart
252 253
		}
		if len(stampStr) < 8 {
254
			return registeredServers, fmt.Errorf("Missing stamp for server [%s] in source from [%v]", name, source.urls)
255
		}
gdm85's avatar
gdm85 committed
256
		stamp, err := stamps.NewServerStampFromString(stampStr)
257
		if err != nil {
Frank Denis's avatar
Frank Denis committed
258
			dlog.Errorf("Invalid or unsupported stamp: [%v]", stampStr)
259 260 261
			return registeredServers, err
		}
		registeredServer := RegisteredServer{
262
			name: name, stamp: stamp, description: description,
263 264 265 266 267 268 269
		}
		dlog.Debugf("Registered [%s] with stamp [%s]", name, stamp.String())
		registeredServers = append(registeredServers, registeredServer)
	}
	return registeredServers, nil
}

270
func PrefetchSourceURL(xTransport *XTransport, urlToPrefetch *URLToPrefetch) error {
271
	in, cached, delayTillNextUpdate, err := fetchWithCache(xTransport, urlToPrefetch.url, urlToPrefetch.cacheFile)
Frank Denis's avatar
Frank Denis committed
272
	if err == nil && !cached {
Frank Denis's avatar
Frank Denis committed
273
		AtomicFileWrite(urlToPrefetch.cacheFile, []byte(in))
274
	}
Frank Denis's avatar
Frank Denis committed
275 276
	urlToPrefetch.when = time.Now().Add(delayTillNextUpdate)
	return err
277
}