Merge remote-tracking branch 'upstream/master' into 278-refactored

Signed-off-by: Krasi Georgiev <kgeorgie@redhat.com>

.travis.yml

@@ -1,5 +1,3 @@
-# sudo is enabled because it provides more memory which was needed to run go test -race
-sudo: required
 dist: trusty
 language: go
 os:
@@ -10,6 +8,7 @@ os:
 go:
   - 1.10.x
   - 1.11.x
+  - 1.12.x
 
 go_import_path: github.com/prometheus/tsdb
 
@@ -20,5 +19,4 @@ install:
   - make deps
 
 script:
-  # `staticcheck` target is omitted due to linting errors
-  - if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then make test; else make; fi
+  - if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then make test; else make all; fi

CHANGELOG.md

@@ -1,11 +1,28 @@
 ## master / unreleased
- - [ENHANCEMENT] Time-ovelapping blocks are now allowed. [#370](https://github.com/prometheus/tsdb/pull/370)
+ - [REMOVED] `chunks.NewReader` is removed as it wasn't used anywhere.
+ - [REMOVED] `FromData` is considered unused so was removed.
+ - [FEATURE] Added option WALSegmentSize -1 to disable the WAL.
+
+## 0.6.1
+  - [BUGFIX] Update `last` after appending a non-overlapping chunk in `chunks.MergeOverlappingChunks`. [#539](https://github.com/prometheus/tsdb/pull/539)
+
+## 0.6.0
+  - [CHANGE] `AllowOverlappingBlock` is now `AllowOverlappingBlocks`.
+
+## 0.5.0
+ - [FEATURE] Time-ovelapping blocks are now allowed. [#370](https://github.com/prometheus/tsdb/pull/370)
+   - Disabled by default and can be enabled via `AllowOverlappingBlock` option.
    - Added `MergeChunks` function in `chunkenc/xor.go` to merge 2 time-overlapping chunks.
    - Added `MergeOverlappingChunks` function in `chunks/chunks.go` to merge multiple time-overlapping Chunk Metas.
    - Added `MinTime` and `MaxTime` method for `BlockReader`.
- - [CHANGE]  `NewLeveledCompactor` takes a context so that a compaction is canceled when closing the db.
+ - [FEATURE] New `dump` command to tsdb tool to dump all samples.
+ - [FEATURE] New `encoding` package for common binary encoding/decoding helpers.
+    - Added to remove some code duplication.
  - [ENHANCEMENT] When closing the db any running compaction will be cancelled so it doesn't block.
- - [CHANGE] `prometheus_tsdb_storage_blocks_bytes_total` is now `prometheus_tsdb_storage_blocks_bytes`
+   - `NewLeveledCompactor` takes a context.
+ - [CHANGE] `prometheus_tsdb_storage_blocks_bytes_total` is now `prometheus_tsdb_storage_blocks_bytes`.
+ - [BUGFIX] Improved Postings Merge performance. Fixes a regression from the the previous release.
+ - [BUGFIX] LiveReader can get into an infinite loop on corrupt WALs.
 
 ## 0.4.0
  - [CHANGE] New `WALSegmentSize` option to override the `DefaultOptions.WALSegmentSize`. Added to allow using smaller wal files. For example using tmpfs on a RPI to minimise the SD card wear out from the constant WAL writes. As part of this change the `DefaultOptions.WALSegmentSize` constant was also exposed.
@@ -25,7 +42,7 @@
  - [CHANGE] `LastCheckpoint()` used to return just the segment name and now it returns the full relative path.
  - [CHANGE] `NewSegmentsRangeReader()` can now read over miltiple wal ranges by using the new `SegmentRange{}` struct.
  - [CHANGE] `CorruptionErr{}` now also exposes the Segment `Dir` which is added when displaying any errors.
- - [CHANGE] `Head.Init()` is changed to `Head.Init(minValidTime int64)`  
+ - [CHANGE] `Head.Init()` is changed to `Head.Init(minValidTime int64)`
  - [CHANGE] `SymbolTable()` renamed to `SymbolTableSize()` to make the name consistent with the  `Block{ symbolTableSize uint64 }` field.
  - [CHANGE] `wal.Reader{}` now exposes `Segment()` for the current segment being read  and `Offset()` for the current offset.
  - [FEATURE] tsdbutil analyze subcomand to find churn, high cardinality, etc.

Makefile

@@ -20,11 +20,6 @@ TSDB_BENCHMARK_OUTPUT_DIR ?= "$(TSDB_CLI_DIR)/benchout"
 
 include Makefile.common
 
-.PHONY: deps
-deps:
-	@echo ">> getting dependencies"
-	GO111MODULE=$(GO111MODULE) $(GO) get $(GOOPTS) -t ./...
-
 build:
 	GO111MODULE=$(GO111MODULE) $(GO) build -o $(TSDB_BIN) $(TSDB_CLI_DIR)
 

Makefile.common

@@ -70,7 +70,7 @@ else
 	GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)
 endif
 
-PROMU_VERSION ?= 0.2.0
+PROMU_VERSION ?= 0.3.0
 PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
 STATICCHECK_VERSION ?= 2019.1
 STATICCHECK_URL     := https://github.com/dominikh/go-tools/releases/download/$(STATICCHECK_VERSION)/staticcheck_$(GOHOSTOS)_$(GOHOSTARCH)
@@ -87,14 +87,14 @@ ifeq ($(GOHOSTARCH),amd64)
         endif
 endif
 
-.PHONY: all
-all: precheck style staticcheck unused build test
-
 # This rule is used to forward a target like "build" to "common-build".  This
 # allows a new "build" target to be defined in a Makefile which includes this
 # one and override "common-build" without override warnings.
 %: common-% ;
 
+.PHONY: common-all
+common-all: precheck style check_license staticcheck unused build test
+
 .PHONY: common-style
 common-style:
 	@echo ">> checking code style"
@@ -116,6 +116,15 @@ common-check_license:
                exit 1; \
        fi
 
+.PHONY: common-deps
+common-deps:
+	@echo ">> getting dependencies"
+ifdef GO111MODULE
+	GO111MODULE=$(GO111MODULE) $(GO) mod download
+else
+	$(GO) get $(GOOPTS) -t ./...
+endif
+
 .PHONY: common-test-short
 common-test-short:
 	@echo ">> running short tests"
@@ -221,7 +230,6 @@ precheck::
 define PRECHECK_COMMAND_template =
 precheck:: $(1)_precheck
 
-
 PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1)))
 .PHONY: $(1)_precheck
 $(1)_precheck:

README.md

@@ -1,5 +1,6 @@
-# TSDB [![Build Status](https://travis-ci.org/prometheus/tsdb.svg?branch=master)](https://travis-ci.org/prometheus/tsdb)
+# TSDB 
 
+[![Build Status](https://travis-ci.org/prometheus/tsdb.svg?branch=master)](https://travis-ci.org/prometheus/tsdb)
 [![GoDoc](https://godoc.org/github.com/prometheus/tsdb?status.svg)](https://godoc.org/github.com/prometheus/tsdb)
 [![Go Report Card](https://goreportcard.com/badge/github.com/prometheus/tsdb)](https://goreportcard.com/report/github.com/prometheus/tsdb)
 

block.go

@@ -28,6 +28,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/tsdb/chunks"
+	tsdb_errors "github.com/prometheus/tsdb/errors"
 	"github.com/prometheus/tsdb/index"
 	"github.com/prometheus/tsdb/labels"
 )
@@ -244,7 +245,7 @@ func writeMetaFile(dir string, meta *BlockMeta) error {
 	enc := json.NewEncoder(f)
 	enc.SetIndent("", "\t")
 
-	var merr MultiError
+	var merr tsdb_errors.MultiError
 
 	if merr.Add(enc.Encode(meta)); merr.Err() != nil {
 		merr.Add(f.Close())
@@ -283,7 +284,7 @@ func OpenBlock(logger log.Logger, dir string, pool chunkenc.Pool) (pb *Block, er
 	var closers []io.Closer
 	defer func() {
 		if err != nil {
-			var merr MultiError
+			var merr tsdb_errors.MultiError
 			merr.Add(err)
 			merr.Add(closeAll(closers))
 			err = merr.Err()
@@ -350,7 +351,7 @@ func (pb *Block) Close() error {
 
 	pb.pendingReaders.Wait()
 
-	var merr MultiError
+	var merr tsdb_errors.MultiError
 
 	merr.Add(pb.chunkr.Close())
 	merr.Add(pb.indexr.Close())

block_test.go

@@ -15,6 +15,8 @@ package tsdb
 
 import (
 	"context"
+	"encoding/binary"
+	"errors"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -23,6 +25,7 @@ import (
 	"testing"
 
 	"github.com/go-kit/kit/log"
+	"github.com/prometheus/tsdb/chunks"
 	"github.com/prometheus/tsdb/testutil"
 	"github.com/prometheus/tsdb/tsdbutil"
 )
@@ -33,7 +36,9 @@ import (
 func TestBlockMetaMustNeverBeVersion2(t *testing.T) {
 	dir, err := ioutil.TempDir("", "metaversion")
 	testutil.Ok(t, err)
-	defer os.RemoveAll(dir)
+	defer func() {
+		testutil.Ok(t, os.RemoveAll(dir))
+	}()
 
 	testutil.Ok(t, writeMetaFile(dir, &BlockMeta{}))
 
@@ -45,7 +50,9 @@ func TestBlockMetaMustNeverBeVersion2(t *testing.T) {
 func TestSetCompactionFailed(t *testing.T) {
 	tmpdir, err := ioutil.TempDir("", "test")
 	testutil.Ok(t, err)
-	defer os.RemoveAll(tmpdir)
+	defer func() {
+		testutil.Ok(t, os.RemoveAll(tmpdir))
+	}()
 
 	blockDir := createBlock(t, tmpdir, genSeries(1, 1, 0, 0, true))
 	b, err := OpenBlock(nil, blockDir, nil)
@@ -61,6 +68,87 @@ func TestSetCompactionFailed(t *testing.T) {
 	testutil.Ok(t, b.Close())
 }
 
+func TestCreateBlock(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "test")
+	testutil.Ok(t, err)
+	defer func() {
+		testutil.Ok(t, os.RemoveAll(tmpdir))
+	}()
+	b, err := OpenBlock(nil, createBlock(t, tmpdir, genSeries(1, 1, 0, 10)), nil)
+	if err == nil {
+		testutil.Ok(t, b.Close())
+	}
+	testutil.Ok(t, err)
+}
+
+func TestCorruptedChunk(t *testing.T) {
+	for name, test := range map[string]struct {
+		corrFunc func(f *os.File) // Func that applies the corruption.
+		expErr   error
+	}{
+		"invalid header size": {
+			func(f *os.File) {
+				err := f.Truncate(1)
+				testutil.Ok(t, err)
+			},
+			errors.New("invalid chunk header in segment 0: invalid size"),
+		},
+		"invalid magic number": {
+			func(f *os.File) {
+				magicChunksOffset := int64(0)
+				_, err := f.Seek(magicChunksOffset, 0)
+				testutil.Ok(t, err)
+
+				// Set invalid magic number.
+				b := make([]byte, chunks.MagicChunksSize)
+				binary.BigEndian.PutUint32(b[:chunks.MagicChunksSize], 0x00000000)
+				n, err := f.Write(b)
+				testutil.Ok(t, err)
+				testutil.Equals(t, chunks.MagicChunksSize, n)
+			},
+			errors.New("invalid magic number 0"),
+		},
+		"invalid chunk format version": {
+			func(f *os.File) {
+				chunksFormatVersionOffset := int64(4)
+				_, err := f.Seek(chunksFormatVersionOffset, 0)
+				testutil.Ok(t, err)
+
+				// Set invalid chunk format version.
+				b := make([]byte, chunks.ChunksFormatVersionSize)
+				b[0] = 0
+				n, err := f.Write(b)
+				testutil.Ok(t, err)
+				testutil.Equals(t, chunks.ChunksFormatVersionSize, n)
+			},
+			errors.New("invalid chunk format version 0"),
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			tmpdir, err := ioutil.TempDir("", "test_open_block_chunk_corrupted")
+			testutil.Ok(t, err)
+			defer func() {
+				testutil.Ok(t, os.RemoveAll(tmpdir))
+			}()
+
+			blockDir := createBlock(t, tmpdir, genSeries(1, 1, 0, 0))
+			files, err := sequenceFiles(chunkDir(blockDir))
+			testutil.Ok(t, err)
+			testutil.Assert(t, len(files) > 0, "No chunk created.")
+
+			f, err := os.OpenFile(files[0], os.O_RDWR, 0666)
+			testutil.Ok(t, err)
+
+			// Apply corruption function.
+			test.corrFunc(f)
+			testutil.Ok(t, f.Close())
+
+			_, err = OpenBlock(nil, blockDir, nil)
+			testutil.Equals(t, test.expErr.Error(), err.Error())
+		})
+	}
+}
+
 // createBlock creates a block with given set of series and returns its dir.
 func createBlock(tb testing.TB, dir string, series []Series) string {
 	head := createHead(tb, series)

checkpoint.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	tsdb_errors "github.com/prometheus/tsdb/errors"
 	"github.com/prometheus/tsdb/fileutil"
 	"github.com/prometheus/tsdb/wal"
 )
@@ -67,7 +68,7 @@ func LastCheckpoint(dir string) (string, int, error) {
 
 // DeleteCheckpoints deletes all checkpoints in a directory below a given index.
 func DeleteCheckpoints(dir string, maxIndex int) error {
-	var errs MultiError
+	var errs tsdb_errors.MultiError
 
 	files, err := ioutil.ReadDir(dir)
 	if err != nil {

checkpoint_test.go

@@ -32,7 +32,9 @@ import (
 func TestLastCheckpoint(t *testing.T) {
 	dir, err := ioutil.TempDir("", "test_checkpoint")
 	testutil.Ok(t, err)
-	defer os.RemoveAll(dir)
+	defer func() {
+		testutil.Ok(t, os.RemoveAll(dir))
+	}()
 
 	_, _, err = LastCheckpoint(dir)
 	testutil.Equals(t, ErrNotFound, err)
@@ -65,7 +67,9 @@ func TestLastCheckpoint(t *testing.T) {
 func TestDeleteCheckpoints(t *testing.T) {
 	dir, err := ioutil.TempDir("", "test_checkpoint")
 	testutil.Ok(t, err)
-	defer os.RemoveAll(dir)
+	defer func() {
+		testutil.Ok(t, os.RemoveAll(dir))
+	}()
 
 	testutil.Ok(t, DeleteCheckpoints(dir, 0))
 
@@ -84,7 +88,9 @@ func TestDeleteCheckpoints(t *testing.T) {
 func TestCheckpoint(t *testing.T) {
 	dir, err := ioutil.TempDir("", "test_checkpoint")
 	testutil.Ok(t, err)
-	defer os.RemoveAll(dir)
+	defer func() {
+		testutil.Ok(t, os.RemoveAll(dir))
+	}()
 
 	var enc RecordEncoder
 	// Create a dummy segment to bump the initial number.
@@ -188,7 +194,9 @@ func TestCheckpointNoTmpFolderAfterError(t *testing.T) {
 	// Create a new wal with an invalid records.
 	dir, err := ioutil.TempDir("", "test_checkpoint")
 	testutil.Ok(t, err)
-	defer os.RemoveAll(dir)
+	defer func() {
+		testutil.Ok(t, os.RemoveAll(dir))
+	}()
 	w, err := wal.NewSize(nil, nil, dir, 64*1024)
 	testutil.Ok(t, err)
 	testutil.Ok(t, w.Log([]byte{99}))

chunkenc/chunk.go

@@ -14,7 +14,6 @@
 package chunkenc
 
 import (
-	"fmt"
 	"sync"
 
 	"github.com/pkg/errors"
@@ -48,15 +47,6 @@ type Chunk interface {
 	NumSamples() int
 }
 
-// FromData returns a chunk from a byte slice of chunk data.
-func FromData(e Encoding, d []byte) (Chunk, error) {
-	switch e {
-	case EncXOR:
-		return &XORChunk{b: bstream{count: 0, stream: d}}, nil
-	}
-	return nil, fmt.Errorf("unknown chunk encoding: %d", e)
-}
-
 // Appender adds sample pairs to a chunk.
 type Appender interface {
 	Append(int64, float64)