diff --git a/cache/async_cache.go b/cache/async_cache.go index ee55e44c..0cafcb4b 100644 --- a/cache/async_cache.go +++ b/cache/async_cache.go @@ -22,6 +22,7 @@ type AsyncCache struct { MaxPayloadSize config.ByteSize SharedWithAllUsers bool + TmpFilePrefix string } func (c *AsyncCache) Close() error { @@ -114,5 +115,6 @@ func NewAsyncCache(cfg config.Cache, maxExecutionTime time.Duration) (*AsyncCach graceTime: graceTime, MaxPayloadSize: maxPayloadSize, SharedWithAllUsers: cfg.SharedWithAllUsers, + TmpFilePrefix: cfg.TmpFilePrefix, }, nil } diff --git a/cache/filesystem_cache_test.go b/cache/filesystem_cache_test.go index fd4da18c..6a3d1223 100644 --- a/cache/filesystem_cache_test.go +++ b/cache/filesystem_cache_test.go @@ -194,7 +194,7 @@ func TestCacheClean(t *testing.T) { Query: []byte(fmt.Sprintf("SELECT %d cache clean", i)), } trw := &testResponseWriter{} - crw, err := NewTmpFileResponseWriter(trw, testTmpWriterDir) + crw, err := NewTmpFileResponseWriter(trw, testTmpWriterDir, DefaultTmpFilePrefix) if err != nil { t.Fatalf("create tmp cache: %s", err) } diff --git a/cache/tmp_file_response_writer.go b/cache/tmp_file_response_writer.go index cab3aad9..be106733 100644 --- a/cache/tmp_file_response_writer.go +++ b/cache/tmp_file_response_writer.go @@ -25,13 +25,18 @@ type TmpFileResponseWriter struct { bw *bufio.Writer // buffered writer for the temporary file } -func NewTmpFileResponseWriter(rw http.ResponseWriter, dir string) (*TmpFileResponseWriter, error) { +const DefaultTmpFilePrefix = "chproxyTmp" + +func NewTmpFileResponseWriter(rw http.ResponseWriter, dir, prefix string) (*TmpFileResponseWriter, error) { _, ok := rw.(http.CloseNotifier) if !ok { return nil, fmt.Errorf("the response writer does not implement http.CloseNotifier") } + if prefix == "" { + prefix = DefaultTmpFilePrefix + } - f, err := os.CreateTemp(dir, "tmp") + f, err := os.CreateTemp(dir, prefix) if err != nil { return nil, fmt.Errorf("cannot create temporary file in %q: %w", dir, err) } diff --git a/cache/tmp_file_response_writer_test.go b/cache/tmp_file_response_writer_test.go index d7d60b92..8cb859a7 100644 --- a/cache/tmp_file_response_writer_test.go +++ b/cache/tmp_file_response_writer_test.go @@ -5,6 +5,8 @@ import ( "log" "net/http" "os" + "path/filepath" + "strings" "testing" ) @@ -60,7 +62,7 @@ func TestFileCreation(t *testing.T) { files, _ := os.ReadDir(testTmpWriterDir) nbFileBefore := len(files) - tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir) + tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, DefaultTmpFilePrefix) defer tmpFileRespWriter.Close() if err != nil { t.Fatalf("could not initate TmpFileResponseWriter error:%s", err) @@ -80,7 +82,7 @@ func TestFileRemoval(t *testing.T) { files, _ := os.ReadDir(testTmpWriterDir) nbFileBefore := len(files) - tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir) + tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, DefaultTmpFilePrefix) if err != nil { t.Fatalf("could not initate TmpFileResponseWriter error:%s", err) return @@ -98,7 +100,7 @@ func TestFileRemoval(t *testing.T) { func TestWriteThenReadHeader(t *testing.T) { srw := newFakeResponse() - tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir) + tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, DefaultTmpFilePrefix) defer tmpFileRespWriter.Close() if err != nil { t.Fatalf("could not initate TmpFileResponseWriter error:%s", err) @@ -129,7 +131,7 @@ func TestWriteThenReadHeader(t *testing.T) { func TestWriteThenReadContent(t *testing.T) { srw := newFakeResponse() - tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir) + tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, DefaultTmpFilePrefix) defer tmpFileRespWriter.Close() if err != nil { t.Fatalf("could not initate TmpFileResponseWriter error:%s", err) @@ -167,7 +169,7 @@ func TestWriteThenReadStatusCode(t *testing.T) { srw := newFakeResponse() expectStatusCode1 := http.StatusOK expectStatusCode2 := 444 - tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir) + tmpFileRespWriter, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, DefaultTmpFilePrefix) defer tmpFileRespWriter.Close() if err != nil { t.Fatalf("could not initate TmpFileResponseWriter error:%s", err) @@ -188,3 +190,19 @@ func TestWriteThenReadStatusCode(t *testing.T) { } } + +func TestTmpFilePrefixIsUsed(t *testing.T) { + const customPrefix = "myCustomPrefix" + srw := newFakeResponse() + + w, err := NewTmpFileResponseWriter(srw, testTmpWriterDir, customPrefix) + if err != nil { + t.Fatalf("could not create TmpFileResponseWriter: %s", err) + } + defer w.Close() + + name := filepath.Base(w.tmpFile.Name()) + if !strings.HasPrefix(name, customPrefix) { + t.Fatalf("expected tmp file name to start with %q, got %q", customPrefix, name) + } +} diff --git a/config/config.go b/config/config.go index a8f94c4e..5f354533 100644 --- a/config/config.go +++ b/config/config.go @@ -944,6 +944,10 @@ type Cache struct { // Whether a query cached by a user could be used by another user SharedWithAllUsers bool `yaml:"shared_with_all_users,omitempty"` + + // Prefix for temporary files created during response caching. + // Defaults to "chproxyTmp" if not set. + TmpFilePrefix string `yaml:"tmp_file_prefix,omitempty"` } func (c *Cache) setDefaults() { diff --git a/config/config_test.go b/config/config_test.go index 76c1e233..5ac7b91e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -31,6 +31,7 @@ var fullConfig = Config{ GraceTime: Duration(20 * time.Second), MaxPayloadSize: ByteSize(100 << 30), SharedWithAllUsers: false, + TmpFilePrefix: "chproxyTmp", }, { Name: "shortterm", @@ -983,3 +984,21 @@ func TestConfigReplaceEnvVars(t *testing.T) { }) } } + +func TestCacheTmpFilePrefixParsed(t *testing.T) { + const input = ` +name: testcache +mode: file_system +tmp_file_prefix: chproxyResponseCache_ +file_system: + dir: /tmp + max_size: 100Mb +` + var c Cache + if err := yaml.Unmarshal([]byte(input), &c); err != nil { + t.Fatalf("unexpected parse error: %s", err) + } + if c.TmpFilePrefix != "chproxyResponseCache_" { + t.Fatalf("expected TmpFilePrefix %q, got %q", "chproxyResponseCache_", c.TmpFilePrefix) + } +} diff --git a/config/testdata/full.yml b/config/testdata/full.yml index 020a5d9f..48187fcc 100644 --- a/config/testdata/full.yml +++ b/config/testdata/full.yml @@ -33,6 +33,13 @@ caches: max_payload_size: 100Gb + # Optional prefix for temporary files created while streaming a response + # into the cache. Useful for distinguishing chproxy temp files from other + # processes when inspecting the OS temp directory. + # + # Defaults to "chproxyTmp". + tmp_file_prefix: chproxyTmp + # Expiration time for cached responses. expire: 1h diff --git a/docs/src/content/docs/configuration/caching.md b/docs/src/content/docs/configuration/caching.md index 67de5733..4b19f971 100644 --- a/docs/src/content/docs/configuration/caching.md +++ b/docs/src/content/docs/configuration/caching.md @@ -63,6 +63,22 @@ User Y will get the cached response from user X's query. Since 1.20.0, the cache is specific for each user by default since it's better in terms of security. It's possible to use the previous behavior by setting the following property of the cache in the config file `shared_with_all_users = true` +#### Temporary file prefix + +While streaming a ClickHouse response into the cache, chproxy writes it to a temporary file in the OS temp directory. The filename starts with a configurable prefix, which defaults to `chproxyTmp`. + +Setting a custom prefix is useful when you need to distinguish chproxy temp files from those of other processes, for example in monitoring or cleanup scripts: + +```yaml +caches: + - name: my-cache + mode: file_system + tmp_file_prefix: chproxyResponseCache_ + file_system: + dir: /path/to/cache + max_size: 100Mb +``` + #### Detecting Cache Hits `Chproxy` will respond with an `X-Cache` header with a value of `HIT` if it returned a response from either the local or the distributed cache. Otherwise `X-Cache` will be set to `MISS`. diff --git a/proxy.go b/proxy.go index 72ec90bb..16c8f7ac 100644 --- a/proxy.go +++ b/proxy.go @@ -418,7 +418,7 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h // The response wasn't found in the cache. // Request it from clickhouse. - tmpFileRespWriter, err := cache.NewTmpFileResponseWriter(srw, os.TempDir()) + tmpFileRespWriter, err := cache.NewTmpFileResponseWriter(srw, os.TempDir(), userCache.TmpFilePrefix) if err != nil { err = fmt.Errorf("%s: %w; query: %q", s, err, q) respondWith(srw, err, http.StatusInternalServerError)