diff --git a/DEVELOPER.md b/DEVELOPER.md index 9334c9b5..535f32bf 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -875,6 +875,49 @@ docker run --rm -it -p 2022:2022 -p 2443:2443 -p 1935:1935 \ Note that the logs should be written to file, there is no log `write log to console`, instead there should be a log like `you can check log by`. +## Go PPROF + +To analyze the performance of Oryx, you can enable the Go pprof tool: + +```bash +GO_PPROF=localhost:6060 go run . +``` + +Run CPU profile: + +```bash +go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30 +``` + +Then use `top` to show the hot functions. + +## Setup for youtube-dl + +Install pyinstaller: + +```bash +brew install pyinstaller +``` + +Clone and build the youtube-dl: + +```bash +cd ~/git && git clone git@github.com:ytdl-org/youtube-dl.git && +cd ~/git/youtube-dl && pyinstaller --onefile --clean --noconfirm --name youtube-dl youtube_dl/__main__.py && +ln -sf ~/git/youtube-dl/dist/youtube-dl /opt/homebrew/bin/ +``` + +Use socks5 proxy for macOS to download: + +```bash +youtube-dl --proxy socks5://127.0.0.1:10000 --output srs 'https://youtu.be/SqrazCPWcV0?si=axNvjynVb7Tf4Bfe' +``` + +> Note: Setup the `--proxy socks5://127.0.0.1:10000` or `YTDL_PROXY=socks5://127.0.0.1:10000` if wants to +> use proxy, use `ssh -D 127.0.0.1:10000 root@x.y.z.m dstat 30` to start the proxy server. + +> Note: Setup the `--output TEMPLATE` when wants to define the filename. + ## WebRTC Candidate Oryx follows the rules for WebRTC candidate, see [CANDIDATE](https://ossrs.io/lts/en-us/docs/v5/doc/webrtc#config-candidate), @@ -1012,6 +1055,7 @@ Platform, with token authentication: * `/terraform/v1/ffmpeg/vlive/source` Setup Virtual Live source file. * `/terraform/v1/ffmpeg/vlive/upload/` Source: Upload Virtual Live or Dubbing source file. * `/terraform/v1/ffmpeg/vlive/server` Source: Use server file as Virtual Live or Dubbing source. +* `/terraform/v1/ffmpeg/vlive/ytdl` Source: Download URL by [youtube-dl](https://github.com/ytdl-org/youtube-dl) as Virtual Live or Dubbing source. * `/terraform/v1/ffmpeg/vlive/stream-url` Source: Use stream URL as Virtual Live source. * `/terraform/v1/ffmpeg/camera/secret` Setup the IP camera streaming secret. * `/terraform/v1/ffmpeg/camera/streams` Query the IP camera streaming streams. @@ -1157,23 +1201,12 @@ Deprecated and unused variables: * `SRS_UTEST`: `on|off`, if on, running in utest mode. * `SOURCE`: `github|gitee`, The source code for upgrading. -Please restart service when `.env` changed. - -## Go PPROF - -To analyze the performance of Oryx, you can enable the Go pprof tool: - -```bash -GO_PPROF=on go run . -``` +Other variables: -Run CPU profile: +* `YTDL_PROXY`: Setup the proxy for youtube-dl, for example, `socks5://127.0.0.1:10000` +* `GO_PPROF`: Setup the listen addr for Go PPROF tool, for example, `localhost:6060` -```bash -go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30 -``` - -Then use `top` to show the hot functions. +Please restart service when `.env` changed. ## Coding Guide @@ -1215,6 +1248,7 @@ The following are the update records for the Oryx server. * VLive: Fix bug when source codec is not supported. v5.15.13 * Forward: Fix high CPU bug. v5.15.14 * Support Go PPROF for CPU profiling. [v5.15.15](https://github.com/ossrs/oryx/releases/tag/v5.15.15) + * VLive: Support download by youtube-dl. v5.15.16 * v5.14: * Merge features and bugfix from releases. v5.14.1 * Dubbing: Support VoD dubbing for multiple languages. [v5.14.2](https://github.com/ossrs/oryx/releases/tag/v5.14.2) diff --git a/platform/dubbing.go b/platform/dubbing.go index 55c05484..46ae2c46 100644 --- a/platform/dubbing.go +++ b/platform/dubbing.go @@ -73,7 +73,7 @@ func handleDubbingService(ctx context.Context, handler *http.ServeMux) error { if targetFile == nil { return errors.Errorf("invalid file") } - if targetFile.Type != FFprobeSourceTypeFile && targetFile.Type != FFprobeSourceTypeUpload { + if targetFile.Type != FFprobeSourceTypeFile && targetFile.Type != FFprobeSourceTypeUpload && targetFile.Type != FFprobeSourceTypeYTDL { return errors.Errorf("invalid file type %v", targetFile.Type) } if targetFile.Target == "" { @@ -2230,7 +2230,7 @@ func (v *SrsDubbingProject) Save(ctx context.Context) error { } func (v *SrsDubbingProject) CheckSource(ctx context.Context, target string) error { - if v.FileType != FFprobeSourceTypeFile && v.FileType != FFprobeSourceTypeUpload { + if v.FileType != FFprobeSourceTypeFile && v.FileType != FFprobeSourceTypeUpload && v.FileType != FFprobeSourceTypeYTDL { return errors.Errorf("unsupported file type %v", v.FileType) } diff --git a/platform/main.go b/platform/main.go index c13dce80..9bd97915 100644 --- a/platform/main.go +++ b/platform/main.go @@ -105,7 +105,7 @@ func doMain(ctx context.Context) error { // Set the default language, en or zh. setEnvDefault("REACT_APP_LOCALE", "en") // Whether enable the Go pprof. - setEnvDefault("GO_PPROF", "off") + setEnvDefault("GO_PPROF", "") // Migrate from mgmt. setEnvDefault("REDIS_DATABASE", "0") @@ -138,8 +138,8 @@ func doMain(ctx context.Context) error { "PUBLIC_URL=%v, BUILD_PATH=%v, REACT_APP_LOCALE=%v, PLATFORM_LISTEN=%v, HTTP_PORT=%v, "+ "REGISTRY=%v, MGMT_LISTEN=%v, HTTPS_LISTEN=%v, AUTO_SELF_SIGNED_CERTIFICATE=%v, "+ "NAME_LOOKUP=%v, PLATFORM_DOCKER=%v, SRS_FORWARD_LIMIT=%v, SRS_VLIVE_LIMIT=%v, "+ - "SRS_CAMERA_LIMIT=%v", - len(envMgmtPassword()), os.Getenv("GO_PPROF"), len(envApiSecret()), envCloud(), + "SRS_CAMERA_LIMIT=%v, YTDL_PROXY=%v", + len(envMgmtPassword()), envGoPprof(), len(envApiSecret()), envCloud(), envRegion(), envSource(), envSrtListen(), envRtcListen(), envNodeEnv(), envLocalRelease(), envRedisDatabase(), envRedisHost(), len(envRedisPassword()), envRedisPort(), @@ -148,16 +148,15 @@ func doMain(ctx context.Context) error { envRegistry(), envMgmtListen(), envHttpListen(), envSelfSignedCertificate(), envNameLookup(), envPlatformDocker(), envForwardLimit(), envVLiveLimit(), - envCameraLimit(), + envCameraLimit(), envYtdlProxy(), ) // Start the Go pprof if enabled. - if os.Getenv("GO_PPROF") == "on" { + if addr := envGoPprof(); addr != "" { go func() { - addr := "localhost:6060" logger.Tf(ctx, "Start Go pprof at %v", addr) http.ListenAndServe(addr, nil) - } () + }() } // Setup the base OS for redis, which should never depends on redis. diff --git a/platform/utils.go b/platform/utils.go index 94f60db1..5a116582 100644 --- a/platform/utils.go +++ b/platform/utils.go @@ -344,6 +344,7 @@ type FFprobeSourceType string const FFprobeSourceTypeUpload FFprobeSourceType = "upload" const FFprobeSourceTypeFile FFprobeSourceType = "file" +const FFprobeSourceTypeYTDL FFprobeSourceType = "ytdl" const FFprobeSourceTypeStream FFprobeSourceType = "stream" // For vLive upload directory. @@ -355,7 +356,7 @@ var dirDubbingPath = path.Join(".", "dub") const serverDataDirectory = "/data" // The video files allowed to use by Oryx. -var serverAllowVideoFiles []string = []string{".mp4", ".flv", ".ts"} +var serverAllowVideoFiles []string = []string{".mp4", ".flv", ".ts", ".mkv", ".mov"} // The audio files allowed to use by Oryx. var serverAllowAudioFiles []string = []string{".mp3", ".aac", ".m4a"} @@ -485,6 +486,14 @@ func envCameraLimit() string { return os.Getenv("SRS_CAMERA_LIMIT") } +func envGoPprof() string { + return os.Getenv("GO_PPROF") +} + +func envYtdlProxy() string { + return os.Getenv("YTDL_PROXY") +} + // rdb is a global redis client object. var rdb *redis.Client diff --git a/platform/virtual-live-stream.go b/platform/virtual-live-stream.go index 2dc55c8d..0d330be1 100644 --- a/platform/virtual-live-stream.go +++ b/platform/virtual-live-stream.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "net/http" "os" "os/exec" @@ -285,6 +286,135 @@ func (v *VLiveWorker) Handle(ctx context.Context, handler *http.ServeMux) error logger.Tf(ctx, "Handle %v", ep) handler.HandleFunc(ep, streamUrlHandler) + ep = "/terraform/v1/ffmpeg/vlive/ytdl" + logger.Tf(ctx, "Handle %v", ep) + handler.HandleFunc(ep, func(w http.ResponseWriter, r *http.Request) { + if err := func() error { + var token string + var qFile string + if err := ParseBody(ctx, r.Body, &struct { + Token *string `json:"token"` + YtdlURL *string `json:"url"` + }{ + Token: &token, YtdlURL: &qFile, + }); err != nil { + return errors.Wrapf(err, "parse body") + } + + apiSecret := envApiSecret() + if err := Authenticate(ctx, apiSecret, token, r.Header); err != nil { + return errors.Wrapf(err, "authenticate") + } + + if !strings.HasPrefix(qFile, "http") && !strings.HasPrefix(qFile, "https") { + return errors.Errorf("invalid url %v", qFile) + } + + // If upload directory is symlink, eval it. + targetDir := dirUploadPath + if info, err := os.Lstat(targetDir); err == nil && info.Mode()&os.ModeSymlink != 0 { + if realPath, err := filepath.EvalSymlinks(targetDir); err != nil { + return errors.Wrapf(err, "eval symlink %v", targetDir) + } else { + targetDir = realPath + } + } + + // The prefix for target files. + targetUUID := uuid.NewString() + + // Cleanup all temporary files created by youtube-dl. + requestDone, requestDoneCancel := context.WithCancel(context.Background()) + defer requestDoneCancel() + go func() { + // If the temporary file still exists for a long time, remove it + duration := 2 * time.Hour + if envNodeEnv() == "development" { + duration = time.Duration(30) * time.Second + } + + select { + case <-ctx.Done(): + logger.Tf(ctx, "ytdl: do cleanup immediately when quit") + case <-requestDone.Done(): + time.Sleep(duration) + } + + filepath.WalkDir(targetDir, func(p string, info fs.DirEntry, err error) error { + if err != nil { + return errors.Wrapf(err, "walk %v", p) + } + + if !info.IsDir() && strings.HasPrefix(info.Name(), targetUUID) { + tempFile := path.Join(dirUploadPath, info.Name()) + if _, err := os.Stat(tempFile); err == nil { + os.Remove(tempFile) + logger.Wf(ctx, "remove %v, duration=%v", tempFile, duration) + } + } + + return nil + }) + }() + + // Use youtube-dl to download the file. + ytdlOutput := path.Join(targetDir, targetUUID) + args := []string{ + "--output", ytdlOutput, + } + if proxy := envYtdlProxy(); proxy != "" { + args = append(args, "--proxy", proxy) + } + args = append(args, qFile) + if err := exec.CommandContext(ctx, "youtube-dl", args...).Run(); err != nil { + return errors.Wrapf(err, "run youtube-dl %v", args) + } + + // Find out the downloaded target file. + var targetFile string + if err := filepath.WalkDir(targetDir, func(p string, info fs.DirEntry, err error) error { + if err != nil { + return errors.Wrapf(err, "walk %v", p) + } + + if !info.IsDir() && strings.HasPrefix(info.Name(), targetUUID) { + targetFile = path.Join(dirUploadPath, info.Name()) + return filepath.SkipAll + } + + return nil + }); err != nil { + return errors.Wrapf(err, "walk %v", targetDir) + } + + if targetFile == "" { + return errors.Errorf("no target file %v", targetUUID) + } + + // Get the file information. + targetFileInfo, err := os.Lstat(targetFile) + if err != nil { + return errors.Wrapf(err, "lstat %v", targetFile) + } + + ohttp.WriteData(ctx, w, r, &struct { + Name string `json:"name"` + UUID string `json:"uuid"` + Target string `json:"target"` + Size int `json:"size"` + }{ + Name: targetFileInfo.Name(), + UUID: targetUUID, + Target: targetFile, + Size: int(targetFileInfo.Size()), + }) + logger.Tf(ctx, "vLive: Got vlive ytdl file target=%v, size=%v", targetFileInfo.Name(), targetFileInfo.Size()) + return nil + }(); err != nil { + ohttp.WriteError(ctx, w, r, err) + } + }) + ep = "/terraform/v1/ffmpeg/vlive/server" logger.Tf(ctx, "Handle %v", ep) handler.HandleFunc(ep, func(w http.ResponseWriter, r *http.Request) { @@ -388,13 +518,22 @@ func (v *VLiveWorker) Handle(ctx context.Context, handler *http.ServeMux) error logger.Wf(ctx, "remove %v, done=%v, created=%v", targetFileName, uploadDone, created) } }() + + requestDone, requestDoneCancel := context.WithCancel(context.Background()) + defer requestDoneCancel() go func() { // If the temporary file still exists for a long time, remove it duration := 2 * time.Hour if envNodeEnv() == "development" { duration = time.Duration(30) * time.Second } - time.Sleep(duration) + + select { + case <-ctx.Done(): + logger.Tf(ctx, "upload: do cleanup immediately when quit") + case <-requestDone.Done(): + time.Sleep(duration) + } if _, err := os.Stat(targetFileName); err == nil { os.Remove(targetFileName) diff --git a/ui/src/components/VideoSourceSelector.js b/ui/src/components/VideoSourceSelector.js index 09f192da..3a3202e9 100644 --- a/ui/src/components/VideoSourceSelector.js +++ b/ui/src/components/VideoSourceSelector.js @@ -5,7 +5,7 @@ // import React from "react"; import {useTranslation} from "react-i18next"; -import {Button, Col, Form, InputGroup, ListGroup, Row} from "react-bootstrap"; +import {Button, Col, Form, InputGroup, ListGroup, Row, Spinner} from "react-bootstrap"; import {SrsErrorBoundary} from "./SrsErrorBoundary"; import {useErrorHandler} from "react-error-boundary"; import axios from "axios"; @@ -19,7 +19,7 @@ export default function ChooseVideoSource({platform, endpoint, vLiveFiles, setVL React.useEffect(() => { if (vLiveFiles?.length) { const type = vLiveFiles[0].type; - if (type === 'upload' || type === 'file' || type === 'stream') { + if (type === 'upload' || type === 'file' || type === 'stream' || type === 'ytdl') { setCheckType(type); } } @@ -50,6 +50,21 @@ export default function ChooseVideoSource({platform, endpoint, vLiveFiles, setVL } + + + setCheckType('ytdl')} + />   + * {t('plat.tool.ytdl20')}.   + {t('helper.see')} link. + + + {checkType === 'ytdl' && + + + + } + {!hideStreamSource && ); } +function YtdlFileSelector({platform, endpoint, vLiveFiles, setVLiveFiles}) { + const {t} = useTranslation(); + const handleError = useErrorHandler(); + // TODO: FIXME: As the file path is changed after used, so we can not use te target. + const [ytdlUrl, setYtdlUrl] = React.useState(''); + const [loading, setLoading] = React.useState(false); + + const CheckYtdlUrl = React.useCallback(async () => { + if (!ytdlUrl) return alert(t('plat.tool.ytdl3')); + if (!ytdlUrl.startsWith('http://') && !ytdlUrl.startsWith('https://')) { + return alert(t('plat.tool.ytdl22')); + } + + try { + setLoading(true); + await new Promise((resolve, reject) => { + axios.post(`/terraform/v1/ffmpeg/vlive/ytdl`, { + url: ytdlUrl, + }, { + headers: Token.loadBearerHeader(), + }).then(res => { + let apiUrl = '/terraform/v1/ffmpeg/vlive/source'; + if (endpoint === 'dubbing') apiUrl = '/terraform/v1/dubbing/source'; + + console.log(`${t('plat.tool.ytdl5')}，${JSON.stringify(res.data.data)}`); + const localFileObj = res.data.data; + const files = [{name: localFileObj.name, path: ytdlUrl, size: localFileObj.size, uuid: localFileObj.uuid, target: localFileObj.target, type: "ytdl"}]; + axios.post(apiUrl, { + platform, files, + }, { + headers: Token.loadBearerHeader(), + }).then(res => { + console.log(`${t('plat.tool.ytdl6')}，${JSON.stringify(res.data.data)}`); + setVLiveFiles(res.data.data.files); + resolve(); + }).catch(reject); + }).catch(reject); + }); + } catch (e) { + handleError(e); + } finally { + setLoading(false); + } + }, [t, ytdlUrl, handleError, platform, setVLiveFiles, endpoint, setLoading]); + + return (<> + + {!vLiveFiles?.length ? <> + + + setYtdlUrl(e.target.value)} /> + + + + {loading && <>  : <> + } + {vLiveFiles?.length ? setVLiveFiles(null)}/> : <>} + + ); +} + function VLiveFileUploader({platform, endpoint, vLiveFiles, setVLiveFiles}) { const {t} = useTranslation(); const handleError = useErrorHandler(); - const updateSources = React.useCallback((platform, files, setFiles) => { + const [loading, setLoading] = React.useState(false); + + const updateSources = React.useCallback(async (platform, files, setFiles) => { if (!files?.length) return alert(t('plat.tool.upload2')); - let apiUrl = '/terraform/v1/ffmpeg/vlive/source'; - if (endpoint === 'dubbing') apiUrl = '/terraform/v1/dubbing/source'; + try { + setLoading(true); + await new Promise((resolve, reject) => { + let apiUrl = '/terraform/v1/ffmpeg/vlive/source'; + if (endpoint === 'dubbing') apiUrl = '/terraform/v1/dubbing/source'; - axios.post(apiUrl, { - platform, files: files.map(f => { - return {name: f.name, path: f.name, size: f.size, uuid: f.uuid, target: f.target, type: "upload"}; - }), - }, { - headers: Token.loadBearerHeader(), - }).then(res => { - console.log(`${t('plat.tool.upload3')}, ${JSON.stringify(res.data.data)}`); - setFiles(res.data.data.files); - }).catch(handleError); - }, [t, handleError, endpoint]); + axios.post(apiUrl, { + platform, files: files.map(f => { + return {name: f.name, path: f.name, size: f.size, uuid: f.uuid, target: f.target, type: "upload"}; + }), + }, { + headers: Token.loadBearerHeader(), + }).then(res => { + console.log(`${t('plat.tool.upload3')}, ${JSON.stringify(res.data.data)}`); + setFiles(res.data.data.files); + resolve(); + }).catch(reject); + }); + } catch (e) { + handleError(e); + } finally { + setLoading(false); + } + }, [t, handleError, endpoint, setLoading]); return (<> {!vLiveFiles?.length ? updateSources(platform, files, setVLiveFiles)}/> : <>} {vLiveFiles?.length ? setVLiveFiles(null)}/> : <>} + {loading && <>  ); } diff --git a/ui/src/resources/locale.json b/ui/src/resources/locale.json index 33030b42..448444e3 100644 --- a/ui/src/resources/locale.json +++ b/ui/src/resources/locale.json @@ -597,6 +597,13 @@ "file5": "检查服务器文件成功", "file6": "更新直播源为服务器文件成功", "file7": "请输入文件路径", + "ytdl": "使用youtube-dl下载视频", + "ytdl20": "支持多个网站的视频", + "ytdl21": "https://ytdl-org.github.io/youtube-dl/supportedsites.html", + "ytdl22": "URL必须是http或https开头", + "ytdl3": "请输入视频URL", + "ytdl5": "检查视频成功", + "ytdl6": "下载视频成功", "stream": "拉流转推", "stream2": "流地址支持 rtmp, srt, http, https, 或 rtsp 等格式", "stream3": "请输入流地址", @@ -757,6 +764,13 @@ "stream3": "Please input stream URL", "stream2": "The stream URL should start with rtmp, srt, http, https, or rtsp.", "stream": "Forward stream", + "ytdl6": "Download video ok", + "ytdl5": "Check video ok", + "ytdl3": "Please input the video URL", + "ytdl22": "Should starts with http or https", + "ytdl21": "https://ytdl-org.github.io/youtube-dl/supportedsites.html", + "ytdl20": "Support not only youtube video", + "ytdl": "Download by youtube-dl", "file7": "Please input file path", "file6": "Setup the live file ok", "file5": "Check server file ok",