Commit a361aa52 authored by Frank Denis's avatar Frank Denis

Preliminary support for remote sources

parent 3824d052
......@@ -31,6 +31,12 @@
packages = ["."]
revision = "6cf43fdfd7a228cf3003ae23d10ddbf65e85997b"
[[projects]]
branch = "master"
name = "github.com/dchest/safefile"
packages = ["."]
revision = "855e8d98f1852d48dde521e0522408d1fe7e836a"
[[projects]]
branch = "master"
name = "github.com/hashicorp/golang-lru"
......@@ -46,6 +52,12 @@
packages = ["."]
revision = "8c253f4161c5b23a5fedd1d1ccee28d7ea312c6c"
[[projects]]
branch = "master"
name = "github.com/jedisct1/go-minisign"
packages = ["."]
revision = "f404c079ea5f0d4669fe617c553651f75167494e"
[[projects]]
branch = "master"
name = "github.com/jedisct1/xsecretbox"
......@@ -87,6 +99,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "03812a1a34033c2d39a6812a33e222c54aaa2f91d01e09072d71b7bd38dceaa3"
inputs-digest = "9946fe30a0b048dbe5b8a10b28e8bffd7ec6dac56380db345cbd868862fe7f08"
solver-name = "gps-cdcl"
solver-version = 1
[[constraint]]
name = "github.com/BurntSushi/toml"
version = "~0.3.0"
version = "0.3.0"
[[constraint]]
name = "github.com/VividCortex/ewma"
version = "~1.1.0"
version = "1.1.1"
[[constraint]]
branch = "master"
name = "github.com/VividCortex/godaemon"
[[constraint]]
branch = "master"
name = "github.com/dchest/safefile"
[[constraint]]
branch = "master"
name = "github.com/hashicorp/golang-lru"
......@@ -18,13 +22,17 @@
branch = "master"
name = "github.com/jedisct1/dlog"
[[constraint]]
branch = "master"
name = "github.com/jedisct1/go-minisign"
[[constraint]]
branch = "master"
name = "github.com/jedisct1/xsecretbox"
[[constraint]]
name = "github.com/miekg/dns"
version = "~1.0.0"
version = "1.0.3"
[[constraint]]
branch = "master"
......
......@@ -24,6 +24,7 @@ type Config struct {
CacheMinTTL uint32 `toml:"cache_min_ttl"`
CacheMaxTTL uint32 `toml:"cache_max_ttl"`
ServersConfig map[string]ServerConfig `toml:"servers"`
SourcesConfig map[string]SourceConfig `toml:"sources"`
}
func newConfig() Config {
......@@ -48,6 +49,14 @@ type ServerConfig struct {
DNSSEC bool `toml:"dnssec"`
}
type SourceConfig struct {
URL string
MinisignKeyStr string `toml:"minisign_key"`
CacheFile string `toml:"cache_file"`
FormatStr string `toml:"format"`
RefreshDelay int `toml:"refresh_delay"`
}
func ConfigLoad(proxy *Proxy, config_file string) error {
configFile := flag.String("config", "dnscrypt-proxy.toml", "path to the configuration file")
flag.Parse()
......@@ -77,8 +86,33 @@ func ConfigLoad(proxy *Proxy, config_file string) error {
config.ServerNames = append(config.ServerNames, serverName)
}
}
if len(config.ServerNames) == 0 {
return errors.New("No servers configured")
for sourceName, source := range config.SourcesConfig {
if source.URL == "" {
return fmt.Errorf("Missing URL for source [%s]", sourceName)
}
if source.MinisignKeyStr == "" {
return fmt.Errorf("Missing Minisign key for source [%s]", sourceName)
}
if source.CacheFile == "" {
return fmt.Errorf("Missing cache file for source [%s]", sourceName)
}
if source.FormatStr == "" {
return fmt.Errorf("Missing format for source [%s]", sourceName)
}
if source.RefreshDelay <= 0 {
source.RefreshDelay = 24
}
source, err := NewSource(source.URL, source.MinisignKeyStr, source.CacheFile, source.FormatStr, time.Duration(source.RefreshDelay)*time.Hour)
if err != nil {
dlog.Criticalf("Unable use source [%s]: [%s]", sourceName, err)
continue
}
registeredServers, err := source.Parse()
if err != nil {
dlog.Criticalf("Unable use source [%s]: [%s]", sourceName, err)
continue
}
proxy.registeredServers = append(proxy.registeredServers, registeredServers...)
}
for _, serverName := range config.ServerNames {
serverConfig, ok := config.ServersConfig[serverName]
......@@ -98,5 +132,8 @@ func ConfigLoad(proxy *Proxy, config_file string) error {
proxy.registeredServers = append(proxy.registeredServers,
RegisteredServer{name: serverName, stamp: stamp})
}
if len(proxy.registeredServers) == 0 {
return errors.New("No servers configured")
}
return nil
}
......@@ -62,7 +62,7 @@ cache_size = 256
## Minimum TTL for cached entries
cache_min_ttl = 60
cache_min_ttl = 600
## Maxmimum TTL for cached entries
......@@ -77,6 +77,15 @@ cache_neg_ttl = 60
############## Servers ##############
## Server sources
[sources]
[sources."github-csv"]
url = "https://raw.githubusercontent.com/DNSCrypt/dnscrypt-resolvers/master/v1/dnscrypt-resolvers.csv"
minisign_key = "RWQf6LRCGA9i53mlYecO4IzT51TGPpvWucNSCh1CBM0QTaLn73Y7GFO3"
cache_file = "/tmp/dnscrypt-resolvers.csv"
format = "v1"
refresh_delay = 24
## Static list of available servers
[servers]
......
......@@ -2,6 +2,7 @@ package main
import (
"encoding/hex"
"fmt"
"math/rand"
"net"
"strings"
......@@ -15,6 +16,7 @@ import (
const (
RTTEwmaDecay = 10.0
DefaultPort = 443
)
type ServerStamp struct {
......@@ -29,6 +31,9 @@ type RegisteredServer struct {
}
func NewServerStampFromLegacy(serverAddrStr string, serverPkStr string, providerName string) (ServerStamp, error) {
if strings.Contains(serverAddrStr, "]") && !strings.Contains(serverAddrStr, ":") {
serverAddrStr = fmt.Sprintf("%s:d", serverAddrStr, DefaultPort)
}
return ServerStamp{
serverAddrStr: serverAddrStr,
serverPkStr: serverPkStr,
......
package main
import (
"encoding/csv"
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"
"time"
"github.com/dchest/safefile"
"github.com/jedisct1/dlog"
"github.com/jedisct1/go-minisign"
)
type SourceFormat int
const (
SourceFormatV1 = iota
)
type Source struct {
url string
format SourceFormat
in string
}
func fetchFromCache(cacheFile string) ([]byte, error) {
dlog.Infof("Loading source information from cache file [%s]", cacheFile)
return ioutil.ReadFile(cacheFile)
}
func fetchWithCache(url string, cacheFile string, refreshDelay time.Duration) (in string, cached bool, err error) {
var bin []byte
cached, usableCache := false, false
fi, err := os.Stat(cacheFile)
if err == nil {
elapsed := time.Now().Sub(fi.ModTime())
if elapsed < refreshDelay && elapsed >= 0 {
usableCache = true
}
}
if usableCache {
bin, err = fetchFromCache(cacheFile)
if err != nil {
return
}
} else {
var resp *http.Response
dlog.Infof("Loading source information from URL [%s]", url)
resp, err = http.Get(url)
if err != nil {
if usableCache {
bin, err = fetchFromCache(cacheFile)
}
if err != nil {
return
}
}
defer resp.Body.Close()
bin, err = ioutil.ReadAll(resp.Body)
if err != nil {
if usableCache {
bin, err = fetchFromCache(cacheFile)
}
if err != nil {
return
}
}
cached = true
}
in = string(bin)
return
}
func AtomicFileWrite(file string, data []byte) error {
return safefile.WriteFile(file, data, 0644)
}
func NewSource(url string, minisignKeyStr string, cacheFile string, formatStr string, refreshDelay time.Duration) (Source, error) {
source := Source{url: url}
if formatStr != "v1" {
return source, fmt.Errorf("Unsupported source format: [%s]", formatStr)
}
source.format = SourceFormatV1
minisignKey, err := minisign.NewPublicKey(minisignKeyStr)
if err != nil {
return source, err
}
in, cached, err := fetchWithCache(url, cacheFile, refreshDelay)
if err != nil {
return source, err
}
sigCacheFile := cacheFile + ".minisig"
sigURL := url + ".minisig"
sigStr, sigCached, err := fetchWithCache(sigURL, sigCacheFile, refreshDelay)
if err != nil {
return source, err
}
signature, err := minisign.DecodeSignature(sigStr)
if err != nil {
return source, err
}
res, err := minisignKey.Verify([]byte(in), signature)
if err != nil || res != true {
return source, err
}
if cached == false {
if err = AtomicFileWrite(cacheFile, []byte(in)); err != nil {
return source, err
}
}
if sigCached == false {
if err = AtomicFileWrite(sigCacheFile, []byte(sigStr)); err != nil {
return source, err
}
}
dlog.Noticef("Source [%s] loaded", url)
source.in = in
return source, nil
}
func (source *Source) Parse() ([]RegisteredServer, error) {
var registeredServers []RegisteredServer
csvReader := csv.NewReader(strings.NewReader(source.in))
records, err := csvReader.ReadAll()
if err != nil {
return registeredServers, nil
}
for line, record := range records {
if len(record) < 14 {
return registeredServers, fmt.Errorf("Parse error at line %d", line)
}
if line == 0 {
continue
}
name := record[0]
serverAddrStr := record[10]
providerName := record[11]
serverPkStr := record[12]
stamp, err := NewServerStampFromLegacy(serverAddrStr, serverPkStr, providerName)
if err != nil {
return registeredServers, err
}
registeredServer := RegisteredServer{
name: name, stamp: stamp,
}
registeredServers = append(registeredServers, registeredServer)
}
return registeredServers, nil
}
language: go
go:
- 1.1
- 1.2
- 1.3
- 1.4
- tip
Copyright (c) 2013 Dmitry Chestnykh <dmitry@codingrobots.com>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials
provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# safefile
[![Build Status](https://travis-ci.org/dchest/safefile.svg)](https://travis-ci.org/dchest/safefile) [![Windows Build status](https://ci.appveyor.com/api/projects/status/owlifxeekg75t2ho?svg=true)](https://ci.appveyor.com/project/dchest/safefile)
Go package safefile implements safe "atomic" saving of files.
Instead of truncating and overwriting the destination file, it creates a
temporary file in the same directory, writes to it, and then renames the
temporary file to the original name when calling Commit.
### Installation
```
$ go get github.com/dchest/safefile
```
### Documentation
<https://godoc.org/github.com/dchest/safefile>
### Example
```go
f, err := safefile.Create("/home/ken/report.txt", 0644)
if err != nil {
// ...
}
// Created temporary file /home/ken/sf-ppcyksu5hyw2mfec.tmp
defer f.Close()
_, err = io.WriteString(f, "Hello world")
if err != nil {
// ...
}
// Wrote "Hello world" to /home/ken/sf-ppcyksu5hyw2mfec.tmp
err = f.Commit()
if err != nil {
// ...
}
// Renamed /home/ken/sf-ppcyksu5hyw2mfec.tmp to /home/ken/report.txt
```
version: "{build}"
os: Windows Server 2012 R2
clone_folder: c:\projects\src\github.com\dchest\safefile
environment:
PATH: c:\projects\bin;%PATH%
GOPATH: c:\projects
NOTIFY_TIMEOUT: 5s
install:
- go version
- go get golang.org/x/tools/cmd/vet
- go get -v -t ./...
build_script:
- go tool vet -all .
- go build ./...
- go test -v -race ./...
test: off
deploy: off
// +build !plan9,!windows windows,go1.5
package safefile
import "os"
func rename(oldname, newname string) error {
return os.Rename(oldname, newname)
}
// +build plan9 windows,!go1.5
// os.Rename on Windows before Go 1.5 and Plan 9 will not overwrite existing
// files, thus we cannot guarantee atomic saving of file by doing rename.
// We will have to do some voodoo to minimize data loss on those systems.
package safefile
import (
"os"
"path/filepath"
)
func rename(oldname, newname string) error {
err := os.Rename(oldname, newname)
if err != nil {
// If newname exists ("original"), we will try renaming it to a
// new temporary name, then renaming oldname to the newname,
// and deleting the renamed original. If system crashes between
// renaming and deleting, the original file will still be available
// under the temporary name, so users can manually recover data.
// (No automatic recovery is possible because after crash the
// temporary name is not known.)
var origtmp string
for {
origtmp, err = makeTempName(newname, filepath.Base(newname))
if err != nil {
return err
}
_, err = os.Stat(origtmp)
if err == nil {
continue // most likely will never happen
}
break
}
err = os.Rename(newname, origtmp)
if err != nil {
return err
}
err = os.Rename(oldname, newname)
if err != nil {
// Rename still fails, try to revert original rename,
// ignoring errors.
os.Rename(origtmp, newname)
return err
}
// Rename succeeded, now delete original file.
os.Remove(origtmp)
}
return nil
}
// Copyright 2013 Dmitry Chestnykh. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package safefile implements safe "atomic" saving of files.
//
// Instead of truncating and overwriting the destination file, it creates a
// temporary file in the same directory, writes to it, and then renames the
// temporary file to the original name when calling Commit.
//
// Example:
//
// f, err := safefile.Create("/home/ken/report.txt", 0644)
// if err != nil {
// // ...
// }
// // Created temporary file /home/ken/sf-ppcyksu5hyw2mfec.tmp
//
// defer f.Close()
//
// _, err = io.WriteString(f, "Hello world")
// if err != nil {
// // ...
// }
// // Wrote "Hello world" to /home/ken/sf-ppcyksu5hyw2mfec.tmp
//
// err = f.Commit()
// if err != nil {
// // ...
// }
// // Renamed /home/ken/sf-ppcyksu5hyw2mfec.tmp to /home/ken/report.txt
//
package safefile
import (
"crypto/rand"
"encoding/base32"
"errors"
"io"
"os"
"path/filepath"
"strings"
)
// ErrAlreadyCommitted error is returned when calling Commit on a file that
// has been already successfully committed.
var ErrAlreadyCommitted = errors.New("file already committed")
type File struct {
*os.File
origName string
closeFunc func(*File) error
isClosed bool // if true, temporary file has been closed, but not renamed
isCommitted bool // if true, the file has been successfully committed
}
func makeTempName(origname, prefix string) (tempname string, err error) {
origname = filepath.Clean(origname)
if len(origname) == 0 || origname[len(origname)-1] == filepath.Separator {
return "", os.ErrInvalid
}
// Generate 10 random bytes.
// This gives 80 bits of entropy, good enough
// for making temporary file name unpredictable.
var rnd [10]byte
if _, err := rand.Read(rnd[:]); err != nil {
return "", err
}
name := prefix + "-" + strings.ToLower(base32.StdEncoding.EncodeToString(rnd[:])) + ".tmp"
return filepath.Join(filepath.Dir(origname), name), nil
}
// Create creates a temporary file in the same directory as filename,
// which will be renamed to the given filename when calling Commit.
func Create(filename string, perm os.FileMode) (*File, error) {
for {
tempname, err := makeTempName(filename, "sf")
if err != nil {
return nil, err
}
f, err := os.OpenFile(tempname, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
if err != nil {
if os.IsExist(err) {
continue
}
return nil, err
}
return &File{
File: f,
origName: filename,
closeFunc: closeUncommitted,
}, nil
}
}
// OrigName returns the original filename given to Create.
func (f *File) OrigName() string {
return f.origName
}
// Close closes temporary file and removes it.
// If the file has been committed, Close is no-op.
func (f *File) Close() error {
return f.closeFunc(f)
}
func closeUncommitted(f *File) error {
err0 := f.File.Close()
err1 := os.Remove(f.Name())
f.closeFunc = closeAgainError
if err0 != nil {
return err0
}
return err1
}
func closeAfterFailedRename(f *File) error {
// Remove temporary file.
//
// The note from Commit function applies here too, as we may be
// removing a different file. However, since we rely on our temporary
// names being unpredictable, this should not be a concern.
f.closeFunc = closeAgainError
return os.Remove(f.Name())
}
func closeCommitted(f *File) error {
// noop
return nil
}
func closeAgainError(f *File) error {
return os.ErrInvalid
}
// Commit safely commits data into the original file by syncing temporary
// file to disk, closing it and renaming to the original file name.
//
// In case of success, the temporary file is closed and no longer exists
// on disk. It is safe to call Close after Commit: the operation will do
// nothing.
//
// In case of error, the temporary file is still opened and exists on disk;
// it must be closed by callers by calling Close or by trying to commit again.
// Note that when trying to Commit again after a failed Commit when the file
// has been closed, but not renamed to its original name (the new commit will
// try again to rename it), safefile cannot guarantee that the temporary file
// has not been changed, or that it is the same temporary file we were dealing
// with. However, since the temporary name is unpredictable, it is unlikely
// that this happened accidentally. If complete atomicity is needed, do not
// Commit again after error, write the file again.
func (f *File) Commit() error {
if f.isCommitted {
return ErrAlreadyCommitted
}
if !f.isClosed {
// Sync to disk.
err := f.Sync()
if err != nil {
return err
}
// Close underlying os.File.
err = f.File.Close()
if err != nil {
return err
}
f.isClosed = true
}
// Rename.
err := rename(f.Name(), f.origName)
if err != nil {
f.closeFunc = closeAfterFailedRename
return err
}
f.closeFunc = closeCommitted
f.isCommitted = true
return nil
}
// WriteFile is a safe analog of ioutil.WriteFile.
func WriteFile(filename string, data []byte, perm os.FileMode) error {
f, err := Create(filename, perm)
if err != nil {
return err
}
defer f.Close()
n, err := f.Write(data)
if err != nil {
return err
}
if err == nil && n < len(data) {
err = io.ErrShortWrite
return err
}
return f.Commit()
}
package safefile
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
)
func ensureFileContains(name, data string) error {
b, err := ioutil.ReadFile(name)
if err != nil {
return err
}
if string(b) != data {
return fmt.Errorf("wrong data in file: expected %s, got %s", data, string(b))
}
return nil
}