VLive: Support download by youtube-dl. v5.15.16

DEVELOPER.md

@@ -875,6 +875,49 @@ docker run --rm -it -p 2022:2022 -p 2443:2443 -p 1935:1935 \
 Note that the logs should be written to file, there is no log `write log to console`, instead there
 should be a log like `you can check log by`.
 
+## Go PPROF
+
+To analyze the performance of Oryx, you can enable the Go pprof tool:
+
+```bash
+GO_PPROF=localhost:6060 go run .
+```
+
+Run CPU profile:
+
+```bash
+go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30
+```
+
+Then use `top` to show the hot functions.
+
+## Setup for youtube-dl
+
+Install pyinstaller:
+
+```bash
+brew install pyinstaller
+```
+
+Clone and build the youtube-dl:
+
+```bash
+cd ~/git && git clone git@github.com:ytdl-org/youtube-dl.git &&
+cd ~/git/youtube-dl && pyinstaller --onefile --clean --noconfirm --name youtube-dl youtube_dl/__main__.py &&
+ln -sf ~/git/youtube-dl/dist/youtube-dl /opt/homebrew/bin/
+```
+
+Use socks5 proxy for macOS to download:
+
+```bash
+youtube-dl --proxy socks5://127.0.0.1:10000 --output srs 'https://youtu.be/SqrazCPWcV0?si=axNvjynVb7Tf4Bfe'
+```
+
+> Note: Setup the `--proxy socks5://127.0.0.1:10000` or `YTDL_PROXY=socks5://127.0.0.1:10000` if wants to
+> use proxy, use `ssh -D 127.0.0.1:10000 root@x.y.z.m dstat 30` to start the proxy server.
+
+> Note: Setup the `--output TEMPLATE` when wants to define the filename.
+
 ## WebRTC Candidate
 
 Oryx follows the rules for WebRTC candidate, see [CANDIDATE](https://ossrs.io/lts/en-us/docs/v5/doc/webrtc#config-candidate),
@@ -1012,6 +1055,7 @@ Platform, with token authentication:
 * `/terraform/v1/ffmpeg/vlive/source` Setup Virtual Live source file.
 * `/terraform/v1/ffmpeg/vlive/upload/` Source: Upload Virtual Live or Dubbing source file.
 * `/terraform/v1/ffmpeg/vlive/server` Source: Use server file as Virtual Live or Dubbing source.
+* `/terraform/v1/ffmpeg/vlive/ytdl` Source: Download URL by [youtube-dl](https://github.com/ytdl-org/youtube-dl) as Virtual Live or Dubbing source.
 * `/terraform/v1/ffmpeg/vlive/stream-url` Source: Use stream URL as Virtual Live source.
 * `/terraform/v1/ffmpeg/camera/secret` Setup the IP camera streaming secret.
 * `/terraform/v1/ffmpeg/camera/streams` Query the IP camera streaming streams.
@@ -1157,23 +1201,12 @@ Deprecated and unused variables:
 * `SRS_UTEST`: `on|off`, if on, running in utest mode.
 * `SOURCE`: `github|gitee`, The source code for upgrading.
 
-Please restart service when `.env` changed.
-
-## Go PPROF
-
-To analyze the performance of Oryx, you can enable the Go pprof tool:
-
-```bash
-GO_PPROF=on go run .
-```
+Other variables:
 
-Run CPU profile:
+* `YTDL_PROXY`: Setup the proxy for youtube-dl, for example, `socks5://127.0.0.1:10000`
+* `GO_PPROF`: Setup the listen addr for Go PPROF tool, for example, `localhost:6060`
 
-```bash
-go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30
-```
-
-Then use `top` to show the hot functions.
+Please restart service when `.env` changed.
 
 ## Coding Guide
 
@@ -1215,6 +1248,7 @@ The following are the update records for the Oryx server.
     * VLive: Fix bug when source codec is not supported. v5.15.13
     * Forward: Fix high CPU bug. v5.15.14
     * Support Go PPROF for CPU profiling. [v5.15.15](https://github.com/ossrs/oryx/releases/tag/v5.15.15)
+    * VLive: Support download by youtube-dl. v5.15.16
 * v5.14:
     * Merge features and bugfix from releases. v5.14.1
     * Dubbing: Support VoD dubbing for multiple languages. [v5.14.2](https://github.com/ossrs/oryx/releases/tag/v5.14.2)

platform/dubbing.go

@@ -73,7 +73,7 @@ func handleDubbingService(ctx context.Context, handler *http.ServeMux) error {
 			if targetFile == nil {
 				return errors.Errorf("invalid file")
 			}
-			if targetFile.Type != FFprobeSourceTypeFile && targetFile.Type != FFprobeSourceTypeUpload {
+			if targetFile.Type != FFprobeSourceTypeFile && targetFile.Type != FFprobeSourceTypeUpload && targetFile.Type != FFprobeSourceTypeYTDL {
 				return errors.Errorf("invalid file type %v", targetFile.Type)
 			}
 			if targetFile.Target == "" {
@@ -2230,7 +2230,7 @@ func (v *SrsDubbingProject) Save(ctx context.Context) error {
 }
 
 func (v *SrsDubbingProject) CheckSource(ctx context.Context, target string) error {
-	if v.FileType != FFprobeSourceTypeFile && v.FileType != FFprobeSourceTypeUpload {
+	if v.FileType != FFprobeSourceTypeFile && v.FileType != FFprobeSourceTypeUpload && v.FileType != FFprobeSourceTypeYTDL {
 		return errors.Errorf("unsupported file type %v", v.FileType)
 	}
 

platform/main.go

@@ -105,7 +105,7 @@ func doMain(ctx context.Context) error {
 	// Set the default language, en or zh.
 	setEnvDefault("REACT_APP_LOCALE", "en")
 	// Whether enable the Go pprof.
-	setEnvDefault("GO_PPROF", "off")
+	setEnvDefault("GO_PPROF", "")
 
 	// Migrate from mgmt.
 	setEnvDefault("REDIS_DATABASE", "0")
@@ -138,8 +138,8 @@ func doMain(ctx context.Context) error {
 		"PUBLIC_URL=%v, BUILD_PATH=%v, REACT_APP_LOCALE=%v, PLATFORM_LISTEN=%v, HTTP_PORT=%v, "+
 		"REGISTRY=%v, MGMT_LISTEN=%v, HTTPS_LISTEN=%v, AUTO_SELF_SIGNED_CERTIFICATE=%v, "+
 		"NAME_LOOKUP=%v, PLATFORM_DOCKER=%v, SRS_FORWARD_LIMIT=%v, SRS_VLIVE_LIMIT=%v, "+
-		"SRS_CAMERA_LIMIT=%v",
-		len(envMgmtPassword()), os.Getenv("GO_PPROF"), len(envApiSecret()), envCloud(),
+		"SRS_CAMERA_LIMIT=%v, YTDL_PROXY=%v",
+		len(envMgmtPassword()), envGoPprof(), len(envApiSecret()), envCloud(),
 		envRegion(), envSource(), envSrtListen(), envRtcListen(),
 		envNodeEnv(), envLocalRelease(),
 		envRedisDatabase(), envRedisHost(), len(envRedisPassword()), envRedisPort(),
@@ -148,16 +148,15 @@ func doMain(ctx context.Context) error {
 		envRegistry(), envMgmtListen(), envHttpListen(),
 		envSelfSignedCertificate(), envNameLookup(),
 		envPlatformDocker(), envForwardLimit(), envVLiveLimit(),
-		envCameraLimit(),
+		envCameraLimit(), envYtdlProxy(),
 	)
 
 	// Start the Go pprof if enabled.
-	if os.Getenv("GO_PPROF") == "on" {
+	if addr := envGoPprof(); addr != "" {
 		go func() {
-			addr := "localhost:6060"
 			logger.Tf(ctx, "Start Go pprof at %v", addr)
 			http.ListenAndServe(addr, nil)
-		} ()
+		}()
 	}
 
 	// Setup the base OS for redis, which should never depends on redis.

platform/utils.go

@@ -344,6 +344,7 @@ type FFprobeSourceType string
 
 const FFprobeSourceTypeUpload FFprobeSourceType = "upload"
 const FFprobeSourceTypeFile FFprobeSourceType = "file"
+const FFprobeSourceTypeYTDL FFprobeSourceType = "ytdl"
 const FFprobeSourceTypeStream FFprobeSourceType = "stream"
 
 // For vLive upload directory.
@@ -355,7 +356,7 @@ var dirDubbingPath = path.Join(".", "dub")
 const serverDataDirectory = "/data"
 
 // The video files allowed to use by Oryx.
-var serverAllowVideoFiles []string = []string{".mp4", ".flv", ".ts"}
+var serverAllowVideoFiles []string = []string{".mp4", ".flv", ".ts", ".mkv", ".mov"}
 
 // The audio files allowed to use by Oryx.
 var serverAllowAudioFiles []string = []string{".mp3", ".aac", ".m4a"}
@@ -485,6 +486,14 @@ func envCameraLimit() string {
 	return os.Getenv("SRS_CAMERA_LIMIT")
 }
 
+func envGoPprof() string {
+	return os.Getenv("GO_PPROF")
+}
+
+func envYtdlProxy() string {
+	return os.Getenv("YTDL_PROXY")
+}
+
 // rdb is a global redis client object.
 var rdb *redis.Client
 

platform/virtual-live-stream.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/http"
 	"os"
 	"os/exec"
@@ -285,6 +286,135 @@ func (v *VLiveWorker) Handle(ctx context.Context, handler *http.ServeMux) error
 	logger.Tf(ctx, "Handle %v", ep)
 	handler.HandleFunc(ep, streamUrlHandler)
 
+	ep = "/terraform/v1/ffmpeg/vlive/ytdl"
+	logger.Tf(ctx, "Handle %v", ep)
+	handler.HandleFunc(ep, func(w http.ResponseWriter, r *http.Request) {
+		if err := func() error {
+			var token string
+			var qFile string
+			if err := ParseBody(ctx, r.Body, &struct {
+				Token   *string `json:"token"`
+				YtdlURL *string `json:"url"`
+			}{
+				Token: &token, YtdlURL: &qFile,
+			}); err != nil {
+				return errors.Wrapf(err, "parse body")
+			}
+
+			apiSecret := envApiSecret()
+			if err := Authenticate(ctx, apiSecret, token, r.Header); err != nil {
+				return errors.Wrapf(err, "authenticate")
+			}
+
+			if !strings.HasPrefix(qFile, "http") && !strings.HasPrefix(qFile, "https") {
+				return errors.Errorf("invalid url %v", qFile)
+			}
+
+			// If upload directory is symlink, eval it.
+			targetDir := dirUploadPath
+			if info, err := os.Lstat(targetDir); err == nil && info.Mode()&os.ModeSymlink != 0 {
+				if realPath, err := filepath.EvalSymlinks(targetDir); err != nil {
+					return errors.Wrapf(err, "eval symlink %v", targetDir)
+				} else {
+					targetDir = realPath
+				}
+			}
+
+			// The prefix for target files.
+			targetUUID := uuid.NewString()
+
+			// Cleanup all temporary files created by youtube-dl.
+			requestDone, requestDoneCancel := context.WithCancel(context.Background())
+			defer requestDoneCancel()
+			go func() {
+				// If the temporary file still exists for a long time, remove it
+				duration := 2 * time.Hour
+				if envNodeEnv() == "development" {
+					duration = time.Duration(30) * time.Second
+				}
+
+				select {
+				case <-ctx.Done():
+					logger.Tf(ctx, "ytdl: do cleanup immediately when quit")
+				case <-requestDone.Done():
+					time.Sleep(duration)
+				}
+
+				filepath.WalkDir(targetDir, func(p string, info fs.DirEntry, err error) error {
+					if err != nil {
+						return errors.Wrapf(err, "walk %v", p)
+					}
+
+					if !info.IsDir() && strings.HasPrefix(info.Name(), targetUUID) {
+						tempFile := path.Join(dirUploadPath, info.Name())
+						if _, err := os.Stat(tempFile); err == nil {
+							os.Remove(tempFile)
+							logger.Wf(ctx, "remove %v, duration=%v", tempFile, duration)
+						}
+					}
+
+					return nil
+				})
+			}()
+
+			// Use youtube-dl to download the file.
+			ytdlOutput := path.Join(targetDir, targetUUID)
+			args := []string{
+				"--output", ytdlOutput,
+			}
+			if proxy := envYtdlProxy(); proxy != "" {
+				args = append(args, "--proxy", proxy)
+			}
+			args = append(args, qFile)
+			if err := exec.CommandContext(ctx, "youtube-dl", args...).Run(); err != nil {
+				return errors.Wrapf(err, "run youtube-dl %v", args)
+			}
+
+			// Find out the downloaded target file.
+			var targetFile string
+			if err := filepath.WalkDir(targetDir, func(p string, info fs.DirEntry, err error) error {
+				if err != nil {
+					return errors.Wrapf(err, "walk %v", p)
+				}
+
+				if !info.IsDir() && strings.HasPrefix(info.Name(), targetUUID) {
+					targetFile = path.Join(dirUploadPath, info.Name())
+					return filepath.SkipAll
+				}
+
+				return nil
+			}); err != nil {
+				return errors.Wrapf(err, "walk %v", targetDir)
+			}
+
+			if targetFile == "" {
+				return errors.Errorf("no target file %v", targetUUID)
+			}
+
+			// Get the file information.
+			targetFileInfo, err := os.Lstat(targetFile)
+			if err != nil {
+				return errors.Wrapf(err, "lstat %v", targetFile)
+			}
+
+			ohttp.WriteData(ctx, w, r, &struct {
+				Name   string `json:"name"`
+				UUID   string `json:"uuid"`
+				Target string `json:"target"`
+				Size   int    `json:"size"`
+			}{
+				Name:   targetFileInfo.Name(),
+				UUID:   targetUUID,
+				Target: targetFile,
+				Size:   int(targetFileInfo.Size()),
+			})
+			logger.Tf(ctx, "vLive: Got vlive ytdl file target=%v, size=%v", targetFileInfo.Name(), targetFileInfo.Size())
+			return nil
+		}(); err != nil {
+			ohttp.WriteError(ctx, w, r, err)
+		}
+	})
+
 	ep = "/terraform/v1/ffmpeg/vlive/server"
 	logger.Tf(ctx, "Handle %v", ep)
 	handler.HandleFunc(ep, func(w http.ResponseWriter, r *http.Request) {
@@ -388,13 +518,22 @@ func (v *VLiveWorker) Handle(ctx context.Context, handler *http.ServeMux) error
 					logger.Wf(ctx, "remove %v, done=%v, created=%v", targetFileName, uploadDone, created)
 				}
 			}()
+
+			requestDone, requestDoneCancel := context.WithCancel(context.Background())
+			defer requestDoneCancel()
 			go func() {
 				// If the temporary file still exists for a long time, remove it
 				duration := 2 * time.Hour
 				if envNodeEnv() == "development" {
 					duration = time.Duration(30) * time.Second
 				}
-				time.Sleep(duration)
+
+				select {
+				case <-ctx.Done():
+					logger.Tf(ctx, "upload: do cleanup immediately when quit")
+				case <-requestDone.Done():
+					time.Sleep(duration)
+				}
 
 				if _, err := os.Stat(targetFileName); err == nil {
 					os.Remove(targetFileName)