diff --git a/Makefile b/Makefile
index ea1ca9e0004..80a327cb844 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@ PROJECT_ROOT=github.com/jaegertracing/jaeger
 TOP_PKGS := $(shell glide novendor | grep -v -e ./thrift-gen/... -e swagger-gen... -e ./examples/... -e ./scripts/...)
 
 # all .go files that don't exist in hidden directories
-ALL_SRC := $(shell find . -name "*.go" | grep -v -e vendor -e thrift-gen -e swagger-gen \
+ALL_SRC := $(shell find . -name "*.go" | grep -v -e vendor -e thrift-gen -e swagger-gen -e examples -e doc.go \
         -e ".*/\..*" \
         -e ".*/_.*" \
         -e ".*/mocks.*")
@@ -41,6 +41,7 @@ SWAGGER_GEN_DIR=swagger-gen
 
 PASS=$(shell printf "\033[32mPASS\033[0m")
 FAIL=$(shell printf "\033[31mFAIL\033[0m")
+FIXME=$(shell printf "\033[31mFIXME\033[0m")
 COLORIZE=$(SED) ''/PASS/s//$(PASS)/'' | $(SED) ''/FAIL/s//$(FAIL)/''
 DOCKER_NAMESPACE?=jaegertracing
 DOCKER_TAG?=latest
@@ -76,6 +77,24 @@ integration-test: go-gen
 storage-integration-test: go-gen
 	$(GOTEST) ./plugin/storage/integration/...
 
+all-pkgs:
+	@echo $(ALL_PKGS) | tr ' ' '\n' | sort
+
+cvr-pkgs:
+	go list $(TOP_PKGS)
+
+.PHONY: cover
+cover: nocover
+	@echo pre-compiling tests
+	@time go test -i $(ALL_PKGS)
+	@./scripts/cover.sh $(shell go list $(TOP_PKGS))
+	go tool cover -html=cover.out -o cover.html
+
+.PHONY: nocover
+nocover:
+	@echo Verifying that all packages have test files to count in coverage
+	@scripts/check-test-files.sh $(subst github.com/jaegertracing/jaeger/,./,$(ALL_PKGS)) | $(SED) ''/FIXME/s//$(FIXME)/''
+
 .PHONY: fmt
 fmt:
 	$(GOFMT) -e -s -l -w $(ALL_SRC)
@@ -172,11 +191,6 @@ build-crossdock: docker-no-ui
 build-crossdock-fresh: build-crossdock-linux
 	make crossdock-fresh
 
-.PHONY: cover
-cover:
-	./scripts/cover.sh $(shell go list $(TOP_PKGS))
-	go tool cover -html=cover.out -o cover.html
-
 .PHONY: install-ci
 install-ci: install
 	go get github.com/wadey/gocovmerge
@@ -186,10 +200,7 @@ install-ci: install
 	go get github.com/sectioneight/md-to-godoc
 
 .PHONY: test-ci
-test-ci: build-examples lint
-	@echo pre-compiling tests
-	@time go test -i $(ALL_PKGS)
-	@./scripts/cover.sh $(shell go list $(TOP_PKGS))
+test-ci: build-examples lint cover
 
 # TODO at the moment we're not generating tchan_*.go files
 .PHONY: thrift
diff --git a/cmd/agent/app/httpserver/thrift-0.9.2/.nocover b/cmd/agent/app/httpserver/thrift-0.9.2/.nocover
new file mode 100644
index 00000000000..152f4437a15
--- /dev/null
+++ b/cmd/agent/app/httpserver/thrift-0.9.2/.nocover
@@ -0,0 +1 @@
+Thrift-generated files
diff --git a/cmd/collector/app/sampling/model/empty_test.go b/cmd/collector/app/sampling/model/empty_test.go
new file mode 100644
index 00000000000..b2766e2b274
--- /dev/null
+++ b/cmd/collector/app/sampling/model/empty_test.go
@@ -0,0 +1,21 @@
+// Copyright (c) 2018 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package model
+
+// import "testing"
+
+// func TestNothing(t *testing.T) {
+// 	// just get the code coverage
+// }
diff --git a/cmd/flags/.nocover b/cmd/flags/.nocover
new file mode 100644
index 00000000000..46911e9377b
--- /dev/null
+++ b/cmd/flags/.nocover
@@ -0,0 +1,2 @@
+FIXME
+
diff --git a/pkg/cassandra/config/.nocover b/pkg/cassandra/config/.nocover
new file mode 100644
index 00000000000..5dfa26a5e8e
--- /dev/null
+++ b/pkg/cassandra/config/.nocover
@@ -0,0 +1 @@
+requires connection to Cassandra
diff --git a/pkg/cassandra/empty_test.go b/pkg/cassandra/empty_test.go
new file mode 100644
index 00000000000..c29238952d6
--- /dev/null
+++ b/pkg/cassandra/empty_test.go
@@ -0,0 +1,15 @@
+// Copyright (c) 2018 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cassandra
diff --git a/pkg/cassandra/gocql/.nocover b/pkg/cassandra/gocql/.nocover
new file mode 100644
index 00000000000..5dfa26a5e8e
--- /dev/null
+++ b/pkg/cassandra/gocql/.nocover
@@ -0,0 +1 @@
+requires connection to Cassandra
diff --git a/pkg/cassandra/gocql/testutils/.nocover b/pkg/cassandra/gocql/testutils/.nocover
new file mode 100644
index 00000000000..9d6cf4b7fb6
--- /dev/null
+++ b/pkg/cassandra/gocql/testutils/.nocover
@@ -0,0 +1 @@
+FIXME
diff --git a/pkg/distributedlock/empty_test.go b/pkg/distributedlock/empty_test.go
new file mode 100644
index 00000000000..452bf0bd1fa
--- /dev/null
+++ b/pkg/distributedlock/empty_test.go
@@ -0,0 +1,15 @@
+// Copyright (c) 2018 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package distributedlock
diff --git a/pkg/es/.nocover b/pkg/es/.nocover
new file mode 100644
index 00000000000..9d6cf4b7fb6
--- /dev/null
+++ b/pkg/es/.nocover
@@ -0,0 +1 @@
+FIXME
diff --git a/pkg/es/config/.nocover b/pkg/es/config/.nocover
new file mode 100644
index 00000000000..517b145dea2
--- /dev/null
+++ b/pkg/es/config/.nocover
@@ -0,0 +1 @@
+requires connection to Elasticsearch
diff --git a/pkg/version/.nocover b/pkg/version/.nocover
new file mode 100644
index 00000000000..46911e9377b
--- /dev/null
+++ b/pkg/version/.nocover
@@ -0,0 +1,2 @@
+FIXME
+
diff --git a/scripts/check-test-files.sh b/scripts/check-test-files.sh
new file mode 100755
index 00000000000..7e44eb99497
--- /dev/null
+++ b/scripts/check-test-files.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -euo pipefail
+
+NO_TEST_FILE_DIRS=""
+for dir in $*; do
+  mainFile=$(find ${dir} -depth 1 -name 'main.go')
+  testFiles=$(find ${dir} -depth 1 -name '*_test.go')
+  if [ -z "${testFiles}" ]; then
+    if [ -n "${mainFile}" ]; then
+      continue # single main does not require tests
+    fi
+    if [ -e ${dir}/.nocover ]; then
+      reason=$(cat ${dir}/.nocover)
+      if [ "${reason}" == "" ]; then 
+        echo "error: ${dir}/.nocover must specify reason" >&2
+        exit 1
+      fi
+      echo "Package excluded from coverage: ${dir}"
+      echo "  reason: ${reason}"
+      continue
+    fi
+    # if [ -e ${dir}/main.go ]; then
+    #   continue
+    # fi
+    if [ -z "${NO_TEST_FILE_DIRS}" ]; then
+      NO_TEST_FILE_DIRS="${dir}"
+    else
+      NO_TEST_FILE_DIRS="${NO_TEST_FILE_DIRS} ${dir}"
+    fi
+  fi
+done
+
+if [ -n "${NO_TEST_FILE_DIRS}" ]; then
+  echo "error: at least one _test.go file must be in all directories with go files so that they are counted for code coverage:" >&2
+  echo ${NO_TEST_FILE_DIRS} | tr ' ' '\n' >&2
+  exit 1
+fi
diff --git a/storage/dependencystore/empty_test.go b/storage/dependencystore/empty_test.go
new file mode 100644
index 00000000000..3f0c2a1a1fd
--- /dev/null
+++ b/storage/dependencystore/empty_test.go
@@ -0,0 +1,15 @@
+// Copyright (c) 2018 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dependencystore
diff --git a/storage/samplingstore/empty_test.go b/storage/samplingstore/empty_test.go
new file mode 100644
index 00000000000..5e467077442
--- /dev/null
+++ b/storage/samplingstore/empty_test.go
@@ -0,0 +1,15 @@
+// Copyright (c) 2018 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package samplingstore