Use os-agnostic os.UserConfigDir func (#114)

* use os agnostic config dir

* introduce config dir migration

* add MigrateConfigFiles func

* add tests+ minor improvements

* fix flag parse warning

* fix lint error

* debug on test failure

* skip test for linux under normal condition

* improve build workflow

* using osutil

* go mod tidy

* use SyncDirectory

* create dir w/ tool name under .config

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
Co-authored-by: Mzack9999 <mzack9999@protonmail.com>
Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>

.github/workflows/build-test.yml

@@ -7,7 +7,6 @@ on:
       - '**.mod'
   workflow_dispatch:
 
-
 jobs:  
   build:
     name: Test Builds
@@ -27,7 +26,8 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Test
-        run: go test ./...
+        run: go test -race ./...
 
-      - name: Build
-        run: go build .
+      - name: Run Example
+        run: go run .
+        working-directory: examples/basic

goflags.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"os"
 	"path"
-	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
@@ -16,6 +15,7 @@ import (
 
 	"github.com/cnf/structhash"
 	fileutil "github.com/projectdiscovery/utils/file"
+	folderutil "github.com/projectdiscovery/utils/folder"
 	permissionutil "github.com/projectdiscovery/utils/permission"
 	"golang.org/x/exp/maps"
 	"gopkg.in/yaml.v3"
@@ -106,20 +106,40 @@ func (flagSet *FlagSet) Parse() error {
 	flagSet.CommandLine.SetOutput(os.Stdout)
 	flagSet.CommandLine.Usage = flagSet.usageFunc
 	_ = flagSet.CommandLine.Parse(os.Args[1:])
+	configFilePath, _ := flagSet.GetConfigFilePath()
 
-	configFilePath, err := flagSet.GetConfigFilePath()
-	if err != nil {
-		return err
-	}
-	_ = os.MkdirAll(filepath.Dir(configFilePath), permissionutil.ConfigFolderPermission)
+	// migrate data from old config dir to new one
+	// Ref: https://github.com/projectdiscovery/nuclei/issues/3576
+	flagSet.migrateConfigDir()
+
+	// if config file doesn't exist, create one
 	if !fileutil.FileExists(configFilePath) {
 		configData := flagSet.generateDefaultConfig()
+		configFileDir := flagSet.GetToolConfigDir()
+		if !fileutil.FolderExists(configFileDir) {
+			_ = fileutil.CreateFolder(configFileDir)
+		}
 		return os.WriteFile(configFilePath, configData, permissionutil.ConfigFilePermission)
 	}
+
 	_ = flagSet.MergeConfigFile(configFilePath) // try to read default config after parsing flags
 	return nil
 }
 
+func (flagSet *FlagSet) migrateConfigDir() {
+	// migration condition
+	// 1. old config dir exists
+	// 2. new config dir doesn't exist
+	// 3. old config dir is not same as new config dir
+
+	toolConfigDir := flagSet.GetToolConfigDir()
+	if toolConfigDir != oldAppConfigDir && fileutil.FolderExists(oldAppConfigDir) && !fileutil.FolderExists(toolConfigDir) {
+		_ = fileutil.CreateFolder(toolConfigDir)
+		// move old config dir to new one
+		_ = folderutil.SyncDirectory(oldAppConfigDir, toolConfigDir)
+	}
+}
+
 // generateDefaultConfig generates a default YAML config file for a flagset.
 func (flagSet *FlagSet) generateDefaultConfig() []byte {
 	hashes := make(map[string]struct{})

goflags_test.go

@@ -5,12 +5,15 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	fileutil "github.com/projectdiscovery/utils/file"
+	osutil "github.com/projectdiscovery/utils/os"
 	permissionutil "github.com/projectdiscovery/utils/permission"
 	"github.com/stretchr/testify/assert"
 
@@ -418,3 +421,69 @@ func tearDown(uniqueValue string) {
 	flag.CommandLine = flag.NewFlagSet(uniqueValue, flag.ContinueOnError)
 	flag.CommandLine.Usage = flag.Usage
 }
+
+func TestConfigDirMigration(t *testing.T) {
+	// remove any args added by previous test
+	os.Args = []string{
+		os.Args[0],
+	}
+	// setup test old config dir
+	createEmptyFilesinDir(t, oldAppConfigDir)
+
+	flagset := NewFlagSet()
+	flagset.CommandLine = flag.NewFlagSet("goflags", flag.ContinueOnError)
+	newToolCfgDir := flagset.GetToolConfigDir()
+
+	// cleanup and debug
+	defer func() {
+		// cleanup
+		if t.Failed() {
+			t.Logf("old config dir: %s", oldAppConfigDir)
+			t.Logf("new config dir: %s", newToolCfgDir)
+			cfgFiles, _ := os.ReadDir(oldAppConfigDir)
+			for _, cfgFile := range cfgFiles {
+				t.Logf("found config file in old dir : %s", cfgFile.Name())
+			}
+			cfgFiles, _ = os.ReadDir(newToolCfgDir)
+			for _, cfgFile := range cfgFiles {
+				t.Logf("found config file in new dir : %s", cfgFile.Name())
+			}
+		}
+		_ = os.RemoveAll(oldAppConfigDir)
+		_ = os.RemoveAll(newToolCfgDir)
+	}()
+
+	// remove new config dir if it already exists from previous test
+	_ = os.RemoveAll(newToolCfgDir)
+
+	// create test flag and parse it
+	var testflag string
+	flagset.CreateGroup("Example", "Example",
+		flagset.StringVarP(&testflag, "test", "t", "", "test flag"),
+	)
+
+	if err := flagset.Parse(); err != nil {
+		require.Nil(t, err, "could not parse flags")
+	}
+
+	// oldAppConfigDir and newConfigDir is same in case of linux (unless sandbox or something else is used)
+	// migration will only happen on windows & macOS (darwin) Ref: https://pkg.go.dev/os#UserConfigDir
+	if oldAppConfigDir != newToolCfgDir || !osutil.IsLinux() {
+		// check if config files are moved to new config dir
+		require.FileExistsf(t, filepath.Join(newToolCfgDir, "config.yaml"), "config file not created in new config dir")
+		require.FileExistsf(t, filepath.Join(newToolCfgDir, "provider-config.yaml"), "config file not created in new config dir")
+	}
+
+	tearDown(t.Name())
+}
+
+func createEmptyFilesinDir(t *testing.T, dirname string) {
+	if !fileutil.FolderExists(dirname) {
+		_ = fileutil.CreateFolder(dirname)
+	}
+	// create empty yaml config files
+	err := os.WriteFile(filepath.Join(oldAppConfigDir, "config.yaml"), []byte{}, os.ModePerm)
+	require.Nil(t, err, "could not create old config file")
+	err = os.WriteFile(filepath.Join(oldAppConfigDir, "provider-config.yaml"), []byte{}, os.ModePerm)
+	require.Nil(t, err, "could not create old config file")
+}

path.go

@@ -4,40 +4,34 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+
+	folderutil "github.com/projectdiscovery/utils/folder"
 )
 
+var oldAppConfigDir = filepath.Join(folderutil.HomeDirOrDefault("."), ".config", getToolName())
+
 // GetConfigFilePath returns the config file path
 func (flagSet *FlagSet) GetConfigFilePath() (string, error) {
 	// return configFilePath if already set
 	if flagSet.configFilePath != "" {
 		return flagSet.configFilePath, nil
 	}
-	// generate default config name
-	appName := filepath.Base(os.Args[0])
-	// trim extension from app name
-	appName = strings.TrimSuffix(appName, filepath.Ext(appName))
-	homePath, err := os.UserHomeDir()
-	if err != nil {
-		return "", err
-	}
-	return filepath.Join(homePath, ".config", appName, "config.yaml"), nil
+	return filepath.Join(folderutil.AppConfigDirOrDefault(".", getToolName()), "config.yaml"), nil
+}
+
+// GetToolConfigDir returns the config dir path of the tool
+func (flagset *FlagSet) GetToolConfigDir() string {
+	cfgFilePath, _ := flagset.GetConfigFilePath()
+	return filepath.Dir(cfgFilePath)
 }
 
 // SetConfigFilePath sets custom config file path
 func (flagSet *FlagSet) SetConfigFilePath(filePath string) {
 	flagSet.configFilePath = filePath
 }
 
-// Deprecated: Use FlagSet.GetConfigFilePath instead.
-// GetConfigFilePath returns the default config file path
-func GetConfigFilePath() (string, error) {
+// getToolName returns the name of the tool
+func getToolName() string {
 	appName := filepath.Base(os.Args[0])
-	// trim extension from app name
-	appName = strings.TrimSuffix(appName, filepath.Ext(appName))
-	homePath, err := os.UserHomeDir()
-	if err != nil {
-		return "", err
-	}
-
-	return filepath.Join(homePath, ".config", appName, "config.yaml"), nil
+	return strings.TrimSuffix(appName, filepath.Ext(appName))
 }