From e4df9bef95e8a4526fe47e9226e615b0efe91b93 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 27 Aug 2024 23:05:28 +0200 Subject: [PATCH] CPLHTTPGetOptionsFromEnv(): fallback to non-streaming filename with no path specific option has been found on the streaming one --- autotest/gcore/vsicurl.py | 41 +++++++++--- port/cpl_http.cpp | 38 +++++++++-- port/cpl_vsi_virtual.h | 6 ++ port/cpl_vsil_curl_streaming.cpp | 106 +++++++++++++++++++++---------- 4 files changed, 142 insertions(+), 49 deletions(-) diff --git a/autotest/gcore/vsicurl.py b/autotest/gcore/vsicurl.py index 099da1e57c5c..95a64748d9c1 100755 --- a/autotest/gcore/vsicurl.py +++ b/autotest/gcore/vsicurl.py @@ -1139,15 +1139,38 @@ def test_vsicurl_GDAL_HTTP_HEADERS(server): filename = ( "/vsicurl/http://localhost:%d/test_vsicurl_GDAL_HTTP_HEADERS.bin" % server.port ) - gdal.SetPathSpecificOption( - filename, - "GDAL_HTTP_HEADERS", - r'Foo: Bar,"Baz: escaped backslash \\, escaped double-quote \", end of value",Another: Header', - ) - with webserver.install_http_handler(handler): - statres = gdal.VSIStatL(filename) - gdal.SetPathSpecificOption(filename, "GDAL_HTTP_HEADERS", None) - assert statres.size == 3 + try: + gdal.SetPathSpecificOption( + filename, + "GDAL_HTTP_HEADERS", + r'Foo: Bar,"Baz: escaped backslash \\, escaped double-quote \", end of value",Another: Header', + ) + with webserver.install_http_handler(handler): + statres = gdal.VSIStatL(filename) + assert statres.size == 3 + + gdal.VSICurlClearCache() + handler = webserver.SequentialHandler() + handler.add( + "HEAD", + "/test_vsicurl_GDAL_HTTP_HEADERS.bin", + 200, + {"Content-Length": "3"}, + expected_headers={ + "Foo": "Bar", + "Baz": r'escaped backslash \, escaped double-quote ", end of value', + "Another": "Header", + }, + ) + with webserver.install_http_handler(handler): + statres = gdal.VSIStatL( + "/vsicurl_streaming/http://localhost:%d/test_vsicurl_GDAL_HTTP_HEADERS.bin" + % server.port + ) + assert statres.size == 3 + + finally: + gdal.SetPathSpecificOption(filename, "GDAL_HTTP_HEADERS", None) ############################################################################### diff --git a/port/cpl_http.cpp b/port/cpl_http.cpp index e64adea82bfe..a754201ec5e5 100644 --- a/port/cpl_http.cpp +++ b/port/cpl_http.cpp @@ -43,6 +43,7 @@ #include "cpl_http.h" #include "cpl_error.h" #include "cpl_multiproc.h" +#include "cpl_vsi_virtual.h" #include "cpl_vsil_curl_class.h" // gcc or clang complains about C-style cast in #define like @@ -554,13 +555,40 @@ constexpr TupleEnvVarOptionName asAssocEnvVarOptionName[] = { char **CPLHTTPGetOptionsFromEnv(const char *pszFilename) { CPLStringList aosOptions; + std::string osNonStreamingFilename; + if (STARTS_WITH(pszFilename, "/vsi")) + { + VSIFilesystemHandler *poFSHandler = + VSIFileManager::GetHandler(pszFilename); + osNonStreamingFilename = + poFSHandler->GetNonStreamingFilename(pszFilename); + if (osNonStreamingFilename == pszFilename) + { + osNonStreamingFilename.clear(); + } + else + { + // CPLDebug("HTTP", "Non-streaming filename for %s: %s", pszFilename, osNonStreamingFilename.c_str()); + } + } for (const auto &sTuple : asAssocEnvVarOptionName) { - const char *pszVal = - pszFilename ? VSIGetPathSpecificOption(pszFilename, - sTuple.pszEnvVar, nullptr) - : CPLGetConfigOption(sTuple.pszEnvVar, nullptr); - if (pszVal != nullptr) + const char *pszVal = nullptr; + if (pszFilename) + { + pszVal = VSIGetPathSpecificOption(pszFilename, sTuple.pszEnvVar, + nullptr); + if (!pszVal && !osNonStreamingFilename.empty()) + { + pszVal = VSIGetPathSpecificOption( + osNonStreamingFilename.c_str(), sTuple.pszEnvVar, nullptr); + } + } + if (!pszVal) + { + pszVal = CPLGetConfigOption(sTuple.pszEnvVar, nullptr); + } + if (pszVal) { aosOptions.AddNameValue(sTuple.pszOptionName, pszVal); } diff --git a/port/cpl_vsi_virtual.h b/port/cpl_vsi_virtual.h index b223c49eb999..5f5e01a0069b 100644 --- a/port/cpl_vsi_virtual.h +++ b/port/cpl_vsi_virtual.h @@ -341,6 +341,12 @@ class CPL_DLL VSIFilesystemHandler return osFilename; } + virtual std::string + GetNonStreamingFilename(const std::string &osFilename) const + { + return osFilename; + } + /** Return the canonical filename. * * May be implemented by case-insensitive filesystems diff --git a/port/cpl_vsil_curl_streaming.cpp b/port/cpl_vsil_curl_streaming.cpp index 10a3824455d9..3fd2fb02f5f7 100644 --- a/port/cpl_vsil_curl_streaming.cpp +++ b/port/cpl_vsil_curl_streaming.cpp @@ -218,7 +218,13 @@ class VSICurlStreamingFSHandler : public VSIFilesystemHandler protected: CPLMutex *hMutex = nullptr; - virtual VSICurlStreamingHandle *CreateFileHandle(const char *pszURL); + virtual VSICurlStreamingHandle *CreateFileHandle(const char *pszFilename, + const char *pszURL); + + virtual std::string GetNonStreamingPrefix() const + { + return "/vsicurl/"; + } public: VSICurlStreamingFSHandler(); @@ -230,11 +236,14 @@ class VSICurlStreamingFSHandler : public VSIFilesystemHandler virtual int Stat(const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags) override; - virtual CPLString GetFSPrefix() + virtual CPLString GetFSPrefix() const { return "/vsicurl_streaming/"; } + std::string + GetNonStreamingFilename(const std::string &osFilename) const override; + const char *GetActualURL(const char *pszFilename) override; const char *GetOptions() override @@ -339,7 +348,8 @@ class VSICurlStreamingHandle : public VSIVirtualHandle void SetURL(const char *pszURL); public: - VSICurlStreamingHandle(VSICurlStreamingFSHandler *poFS, const char *pszURL); + VSICurlStreamingHandle(VSICurlStreamingFSHandler *poFS, + const char *pszFilename, const char *pszURL); ~VSICurlStreamingHandle() override; int Seek(vsi_l_offset nOffset, int nWhence) override; @@ -380,9 +390,9 @@ class VSICurlStreamingHandle : public VSIVirtualHandle /************************************************************************/ VSICurlStreamingHandle::VSICurlStreamingHandle(VSICurlStreamingFSHandler *poFS, + const char *pszFilename, const char *pszURL) - : m_poFS(poFS), m_aosHTTPOptions(CPLHTTPGetOptionsFromEnv( - (poFS->GetFSPrefix() + pszURL).c_str())), + : m_poFS(poFS), m_aosHTTPOptions(CPLHTTPGetOptionsFromEnv(pszFilename)), m_oRetryParameters(m_aosHTTPOptions), m_pszURL(CPLStrdup(pszURL)) { FileProp cachedFileProp; @@ -1595,9 +1605,10 @@ void VSICurlStreamingFSHandler::ReleaseMutex() /************************************************************************/ VSICurlStreamingHandle * -VSICurlStreamingFSHandler::CreateFileHandle(const char *pszURL) +VSICurlStreamingFSHandler::CreateFileHandle(const char *pszFilename, + const char *pszURL) { - return new VSICurlStreamingHandle(this, pszURL); + return new VSICurlStreamingHandle(this, pszFilename, pszURL); } /************************************************************************/ @@ -1621,7 +1632,7 @@ VSIVirtualHandle *VSICurlStreamingFSHandler::Open(const char *pszFilename, } VSICurlStreamingHandle *poHandle = - CreateFileHandle(pszFilename + GetFSPrefix().size()); + CreateFileHandle(pszFilename, pszFilename + GetFSPrefix().size()); // If we didn't get a filelist, check that the file really exists. if (poHandle == nullptr || !poHandle->Exists(pszFilename, papszOptions)) { @@ -1687,7 +1698,7 @@ int VSICurlStreamingFSHandler::Stat(const char *pszFilename, memset(pStatBuf, 0, sizeof(VSIStatBufL)); VSICurlStreamingHandle *poHandle = - CreateFileHandle(pszFilename + GetFSPrefix().size()); + CreateFileHandle(pszFilename, pszFilename + GetFSPrefix().size()); if (poHandle == nullptr) { return -1; @@ -1715,12 +1726,25 @@ const char *VSICurlStreamingFSHandler::GetActualURL(const char *pszFilename) if (!STARTS_WITH_CI(pszFilename, GetFSPrefix())) return pszFilename; auto poHandle = std::unique_ptr( - CreateFileHandle(pszFilename + GetFSPrefix().size())); + CreateFileHandle(pszFilename, pszFilename + GetFSPrefix().size())); if (poHandle == nullptr) return pszFilename; return CPLSPrintf("%s", poHandle->GetURL()); } +/************************************************************************/ +/* GetNonStreamingFilename() */ +/************************************************************************/ + +std::string VSICurlStreamingFSHandler::GetNonStreamingFilename( + const std::string &osFilename) const +{ + if (STARTS_WITH(osFilename.c_str(), GetFSPrefix().c_str())) + return GetNonStreamingPrefix() + + osFilename.substr(GetFSPrefix().size()); + return osFilename; +} + /************************************************************************/ /* IVSIS3LikeStreamingFSHandler */ /************************************************************************/ @@ -1729,9 +1753,6 @@ class IVSIS3LikeStreamingFSHandler : public VSICurlStreamingFSHandler { CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeStreamingFSHandler) - protected: - virtual std::string GetNonStreamingPrefix() const = 0; - public: IVSIS3LikeStreamingFSHandler() = default; @@ -1762,7 +1783,7 @@ class VSIS3StreamingFSHandler final : public IVSIS3LikeStreamingFSHandler CPL_DISALLOW_COPY_ASSIGN(VSIS3StreamingFSHandler) protected: - CPLString GetFSPrefix() override + CPLString GetFSPrefix() const override { return "/vsis3_streaming/"; } @@ -1772,7 +1793,8 @@ class VSIS3StreamingFSHandler final : public IVSIS3LikeStreamingFSHandler return "/vsis3/"; } - VSICurlStreamingHandle *CreateFileHandle(const char *pszURL) override; + VSICurlStreamingHandle *CreateFileHandle(const char *pszFilename, + const char *pszURL) override; public: VSIS3StreamingFSHandler() = default; @@ -1815,6 +1837,7 @@ class VSIS3LikeStreamingHandle final : public VSICurlStreamingHandle public: VSIS3LikeStreamingHandle(IVSIS3LikeStreamingFSHandler *poFS, + const char *pszFilename, IVSIS3LikeHandleHelper *poS3HandleHelper); ~VSIS3LikeStreamingHandle() override; }; @@ -1824,13 +1847,15 @@ class VSIS3LikeStreamingHandle final : public VSICurlStreamingHandle /************************************************************************/ VSICurlStreamingHandle * -VSIS3StreamingFSHandler::CreateFileHandle(const char *pszURL) +VSIS3StreamingFSHandler::CreateFileHandle(const char *pszFilename, + const char *pszURL) { VSIS3HandleHelper *poS3HandleHelper = VSIS3HandleHelper::BuildFromURI(pszURL, GetFSPrefix().c_str(), false); if (poS3HandleHelper) { - return new VSIS3LikeStreamingHandle(this, poS3HandleHelper); + return new VSIS3LikeStreamingHandle(this, pszFilename, + poS3HandleHelper); } return nullptr; } @@ -1840,9 +1865,10 @@ VSIS3StreamingFSHandler::CreateFileHandle(const char *pszURL) /************************************************************************/ VSIS3LikeStreamingHandle::VSIS3LikeStreamingHandle( - IVSIS3LikeStreamingFSHandler *poFS, + IVSIS3LikeStreamingFSHandler *poFS, const char *pszFilename, IVSIS3LikeHandleHelper *poS3HandleHelper) - : VSICurlStreamingHandle(poFS, poS3HandleHelper->GetURL().c_str()), + : VSICurlStreamingHandle(poFS, pszFilename, + poS3HandleHelper->GetURL().c_str()), m_poS3HandleHelper(poS3HandleHelper) { } @@ -1890,7 +1916,7 @@ bool VSIS3LikeStreamingHandle::CanRestartOnError(const char *pszErrorMsg, class VSIGSStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler { protected: - CPLString GetFSPrefix() override + CPLString GetFSPrefix() const override { return "/vsigs_streaming/"; } @@ -1900,7 +1926,8 @@ class VSIGSStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler return "/vsigs/"; } - VSICurlStreamingHandle *CreateFileHandle(const char *pszURL) override; + VSICurlStreamingHandle *CreateFileHandle(const char *pszFilename, + const char *pszURL) override; public: VSIGSStreamingFSHandler() @@ -1917,13 +1944,15 @@ class VSIGSStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler /************************************************************************/ VSICurlStreamingHandle * -VSIGSStreamingFSHandler::CreateFileHandle(const char *pszURL) +VSIGSStreamingFSHandler::CreateFileHandle(const char *pszFilename, + const char *pszURL) { VSIGSHandleHelper *poGCHandleHelper = VSIGSHandleHelper::BuildFromURI(pszURL, GetFSPrefix().c_str()); if (poGCHandleHelper) { - return new VSIS3LikeStreamingHandle(this, poGCHandleHelper); + return new VSIS3LikeStreamingHandle(this, pszFilename, + poGCHandleHelper); } return nullptr; } @@ -1935,7 +1964,7 @@ VSIGSStreamingFSHandler::CreateFileHandle(const char *pszURL) class VSIAzureStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler { protected: - CPLString GetFSPrefix() override + CPLString GetFSPrefix() const override { return "/vsiaz_streaming/"; } @@ -1945,7 +1974,8 @@ class VSIAzureStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler return "/vsiaz/"; } - VSICurlStreamingHandle *CreateFileHandle(const char *pszURL) override; + VSICurlStreamingHandle *CreateFileHandle(const char *pszFilename, + const char *pszURL) override; public: VSIAzureStreamingFSHandler() @@ -1962,13 +1992,14 @@ class VSIAzureStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler /************************************************************************/ VSICurlStreamingHandle * -VSIAzureStreamingFSHandler::CreateFileHandle(const char *pszURL) +VSIAzureStreamingFSHandler::CreateFileHandle(const char *pszFilename, + const char *pszURL) { VSIAzureBlobHandleHelper *poHandleHelper = VSIAzureBlobHandleHelper::BuildFromURI(pszURL, GetFSPrefix().c_str()); if (poHandleHelper) { - return new VSIS3LikeStreamingHandle(this, poHandleHelper); + return new VSIS3LikeStreamingHandle(this, pszFilename, poHandleHelper); } return nullptr; } @@ -1982,7 +2013,7 @@ class VSIOSSStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler CPL_DISALLOW_COPY_ASSIGN(VSIOSSStreamingFSHandler) protected: - CPLString GetFSPrefix() override + CPLString GetFSPrefix() const override { return "/vsioss_streaming/"; } @@ -1992,7 +2023,8 @@ class VSIOSSStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler return "/vsioss/"; } - VSICurlStreamingHandle *CreateFileHandle(const char *pszURL) override; + VSICurlStreamingHandle *CreateFileHandle(const char *pszFilename, + const char *pszURL) override; public: VSIOSSStreamingFSHandler() = default; @@ -2010,13 +2042,15 @@ class VSIOSSStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler /************************************************************************/ VSICurlStreamingHandle * -VSIOSSStreamingFSHandler::CreateFileHandle(const char *pszURL) +VSIOSSStreamingFSHandler::CreateFileHandle(const char *pszFilename, + const char *pszURL) { VSIOSSHandleHelper *poOSSHandleHelper = VSIOSSHandleHelper::BuildFromURI(pszURL, GetFSPrefix().c_str(), false); if (poOSSHandleHelper) { - return new VSIS3LikeStreamingHandle(this, poOSSHandleHelper); + return new VSIS3LikeStreamingHandle(this, pszFilename, + poOSSHandleHelper); } return nullptr; } @@ -2028,7 +2062,7 @@ VSIOSSStreamingFSHandler::CreateFileHandle(const char *pszURL) class VSISwiftStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler { protected: - CPLString GetFSPrefix() override + CPLString GetFSPrefix() const override { return "/vsiswift_streaming/"; } @@ -2038,7 +2072,8 @@ class VSISwiftStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler return "/vsiswift/"; } - VSICurlStreamingHandle *CreateFileHandle(const char *pszURL) override; + VSICurlStreamingHandle *CreateFileHandle(const char *pszFilename, + const char *pszURL) override; public: VSISwiftStreamingFSHandler() @@ -2055,13 +2090,14 @@ class VSISwiftStreamingFSHandler final : public IVSIS3LikeStreamingFSHandler /************************************************************************/ VSICurlStreamingHandle * -VSISwiftStreamingFSHandler::CreateFileHandle(const char *pszURL) +VSISwiftStreamingFSHandler::CreateFileHandle(const char *pszFilename, + const char *pszURL) { VSISwiftHandleHelper *poHandleHelper = VSISwiftHandleHelper::BuildFromURI(pszURL, GetFSPrefix().c_str()); if (poHandleHelper) { - return new VSIS3LikeStreamingHandle(this, poHandleHelper); + return new VSIS3LikeStreamingHandle(this, pszFilename, poHandleHelper); } return nullptr; }