Merge branch 'master' of gitlab.alibaba-inc.com:xr-paas/gradio_webrtc

frontend/gradio.config.js

@@ -1,9 +1,25 @@
+// export default {
+//   plugins: [],
+//   svelte: {
+//     preprocess: [],
+//   },
+//   build: {
+//     target: "modules",
+//   },
+// };
+
+import commonjs from "vite-plugin-commonjs";
+
 export default {
-  plugins: [],
+  plugins: [
+    commonjs({
+      filter(id) { return id.includes("node_modules/deepmerge"); }
+    })
+  ],
   svelte: {
     preprocess: [],
   },
   build: {
     target: "modules",
   },
-};
+};

frontend/package-lock.json

@@ -25,7 +25,8 @@
       },
       "devDependencies": {
         "@gradio/preview": "0.12.0",
-        "prettier": "3.3.3"
+        "prettier": "3.3.3",
+        "vite-plugin-commonjs": "^0.10.4"
       },
       "peerDependencies": {
         "svelte": "^4.0.0"
@@ -982,6 +983,44 @@
         "node": ">=18"
       }
     },
+    "node_modules/@nodelib/fs.scandir": {
+      "version": "2.1.5",
+      "resolved": "https://registry.anpm.alibaba-inc.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@nodelib/fs.stat": "2.0.5",
+        "run-parallel": "^1.1.9"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@nodelib/fs.stat": {
+      "version": "2.0.5",
+      "resolved": "https://registry.anpm.alibaba-inc.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@nodelib/fs.walk": {
+      "version": "1.2.8",
+      "resolved": "https://registry.anpm.alibaba-inc.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@nodelib/fs.scandir": "2.1.5",
+        "fastq": "^1.6.0"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/@open-draft/deferred-promise": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz",
@@ -2378,6 +2417,13 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/es-module-lexer": {
+      "version": "1.6.0",
+      "resolved": "https://registry.anpm.alibaba-inc.com/es-module-lexer/-/es-module-lexer-1.6.0.tgz",
+      "integrity": "sha512-qqnD1yMU6tk/jnaMosogGySTZP8YtUgAffA9nMN+E/rjxcfRQ6IEk7IiozUjgxKoFHBGjTLnrHB/YC45r/59EQ==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/es5-ext": {
       "version": "0.10.64",
       "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.64.tgz",
@@ -2851,6 +2897,33 @@
         "type": "^2.7.2"
       }
     },
+    "node_modules/fast-glob": {
+      "version": "3.3.3",
+      "resolved": "https://registry.anpm.alibaba-inc.com/fast-glob/-/fast-glob-3.3.3.tgz",
+      "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@nodelib/fs.stat": "^2.0.2",
+        "@nodelib/fs.walk": "^1.2.3",
+        "glob-parent": "^5.1.2",
+        "merge2": "^1.3.0",
+        "micromatch": "^4.0.8"
+      },
+      "engines": {
+        "node": ">=8.6.0"
+      }
+    },
+    "node_modules/fastq": {
+      "version": "1.18.0",
+      "resolved": "https://registry.anpm.alibaba-inc.com/fastq/-/fastq-1.18.0.tgz",
+      "integrity": "sha512-QKHXPW0hD8g4UET03SdOdunzSouc9N4AuHdsX8XNcTsuz+yYFILVNIX4l9yHABMhiEI9Db0JTTIpu0wB+Y1QQw==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "reusify": "^1.0.4"
+      }
+    },
     "node_modules/fetch-event-stream": {
       "version": "0.1.5",
       "resolved": "https://registry.npmjs.org/fetch-event-stream/-/fetch-event-stream-0.1.5.tgz",
@@ -2979,6 +3052,19 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/glob-parent": {
+      "version": "5.1.2",
+      "resolved": "https://registry.anpm.alibaba-inc.com/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "is-glob": "^4.0.1"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/globalyzer": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/globalyzer/-/globalyzer-0.1.0.tgz",
@@ -3739,6 +3825,16 @@
         "node": ">=0.12"
       }
     },
+    "node_modules/merge2": {
+      "version": "1.4.1",
+      "resolved": "https://registry.anpm.alibaba-inc.com/merge2/-/merge2-1.4.1.tgz",
+      "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/micromatch": {
       "version": "4.0.8",
       "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
@@ -4298,6 +4394,27 @@
       "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
       "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
     },
+    "node_modules/queue-microtask": {
+      "version": "1.2.3",
+      "resolved": "https://registry.anpm.alibaba-inc.com/queue-microtask/-/queue-microtask-1.2.3.tgz",
+      "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "license": "MIT"
+    },
     "node_modules/readdirp": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.0.2.tgz",
@@ -4346,6 +4463,17 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/reusify": {
+      "version": "1.0.4",
+      "resolved": "https://registry.anpm.alibaba-inc.com/reusify/-/reusify-1.0.4.tgz",
+      "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "iojs": ">=1.0.0",
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/rimraf": {
       "version": "2.7.1",
       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
@@ -4399,6 +4527,30 @@
       "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.7.1.tgz",
       "integrity": "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg=="
     },
+    "node_modules/run-parallel": {
+      "version": "1.2.0",
+      "resolved": "https://registry.anpm.alibaba-inc.com/run-parallel/-/run-parallel-1.2.0.tgz",
+      "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "queue-microtask": "^1.2.2"
+      }
+    },
     "node_modules/sade": {
       "version": "1.8.1",
       "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz",
@@ -5267,6 +5419,57 @@
         }
       }
     },
+    "node_modules/vite-plugin-commonjs": {
+      "version": "0.10.4",
+      "resolved": "https://registry.anpm.alibaba-inc.com/vite-plugin-commonjs/-/vite-plugin-commonjs-0.10.4.tgz",
+      "integrity": "sha512-eWQuvQKCcx0QYB5e5xfxBNjQKyrjEWZIR9UOkOV6JAgxVhtbZvCOF+FNC2ZijBJ3U3Px04ZMMyyMyFBVWIJ5+g==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "acorn": "^8.12.1",
+        "magic-string": "^0.30.11",
+        "vite-plugin-dynamic-import": "^1.6.0"
+      }
+    },
+    "node_modules/vite-plugin-commonjs/node_modules/acorn": {
+      "version": "8.14.0",
+      "resolved": "https://registry.anpm.alibaba-inc.com/acorn/-/acorn-8.14.0.tgz",
+      "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==",
+      "dev": true,
+      "license": "MIT",
+      "bin": {
+        "acorn": "bin/acorn"
+      },
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/vite-plugin-dynamic-import": {
+      "version": "1.6.0",
+      "resolved": "https://registry.anpm.alibaba-inc.com/vite-plugin-dynamic-import/-/vite-plugin-dynamic-import-1.6.0.tgz",
+      "integrity": "sha512-TM0sz70wfzTIo9YCxVFwS8OA9lNREsh+0vMHGSkWDTZ7bgd1Yjs5RV8EgB634l/91IsXJReg0xtmuQqP0mf+rg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "acorn": "^8.12.1",
+        "es-module-lexer": "^1.5.4",
+        "fast-glob": "^3.3.2",
+        "magic-string": "^0.30.11"
+      }
+    },
+    "node_modules/vite-plugin-dynamic-import/node_modules/acorn": {
+      "version": "8.14.0",
+      "resolved": "https://registry.anpm.alibaba-inc.com/acorn/-/acorn-8.14.0.tgz",
+      "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==",
+      "dev": true,
+      "license": "MIT",
+      "bin": {
+        "acorn": "bin/acorn"
+      },
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
     "node_modules/vite/node_modules/@esbuild/aix-ppc64": {
       "version": "0.21.5",
       "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz",

frontend/package.json

@@ -23,7 +23,8 @@
   },
   "devDependencies": {
     "@gradio/preview": "0.12.0",
-    "prettier": "3.3.3"
+    "prettier": "3.3.3",
+     "vite-plugin-commonjs": "^0.10.4"
   },
   "exports": {
     "./package.json": "./package.json",

frontend/shared/Webcam.svelte

@@ -20,6 +20,7 @@
     import { start, stop } from "./webrtc_utils";
     import PulsingIcon from "./PulsingIcon.svelte";
 
+	let local_video_source: HTMLVideoElement;
 	let video_source: HTMLVideoElement;
 	let available_video_devices: MediaDeviceInfo[] = [];
 	let available_audio_devices: MediaDeviceInfo[] = [];
@@ -61,6 +62,8 @@
 	export let include_audio: boolean;
 	export let i18n: I18nFormatter;
 
+	let isKeepLocal = mode === "send-receive" && include_audio
+
 	const dispatch = createEventDispatcher<{
 		tick: undefined;
 		error: string;
@@ -81,10 +84,11 @@
 		} else {
 			videoDeviceId = device_id
 		}
-
+		const node = isKeepLocal ? local_video_source : video_source; 
 		await get_video_stream(audioDeviceId ? {
 				deviceId: { exact: audioDeviceId },
-		}: include_audio, video_source, videoDeviceId, track_constraints).then(
+		}: include_audio, node, videoDeviceId, track_constraints).then(
+
 			async (local_stream) => {
 				stream = local_stream;
 				selected_device =
@@ -103,7 +107,8 @@
 
 	async function access_webcam(): Promise<void> {
 		try {
-			get_video_stream(include_audio, video_source, null, track_constraints)
+			const node = isKeepLocal ? local_video_source : video_source; 
+			get_video_stream(include_audio, node, null, track_constraints)
 				.then(async (local_stream) => {
 					webcam_accessed = true;
 					let available_devices = await get_devices();
@@ -147,6 +152,7 @@
 	let stream: MediaStream;
 
 	let webcam_accessed = false;
+	let webcam_received = false;
     let pc: RTCPeerConnection;
 	export let webrtc_id;
 
@@ -175,14 +181,17 @@
 			webrtc_id = Math.random().toString(36).substring(2);
             start(stream, pc, mode === "send" ? null: video_source, server.offer, webrtc_id, "video", on_change_cb, rtp_params).then((connection) => {
 				pc = connection;
+				webcam_received = true;
 			}).catch(() => {
                 console.info("catching")
                 stream_state = "closed";
+				webcam_received = false;
                 dispatch("error", "Too many concurrent users. Come back later!");
             });
         } else {
             stop(pc);
             stream_state = "closed";
+			webcam_received = false;
 			_time_limit = null;
             await access_webcam();
         }
@@ -234,13 +243,35 @@
 	{/if}
 	<!-- svelte-ignore a11y-media-has-caption -->
 	<!-- need to suppress for video streaming https://github.com/sveltejs/svelte/issues/5967 -->
-	<video
+	 {#if isKeepLocal}
-		bind:this={video_source}
+	 	<div class="video-wrap">
-		class:hide={!webcam_accessed}
+        	<video
-        class:flip={(stream_state != "open") || (stream_state === "open" && include_audio)}
+            	bind:this={local_video_source}
-        autoplay={true}
+            	class="original-video-source"
-		playsinline={true}
+            	class:hide={!webcam_accessed}
-	/>
+            	autoplay={true}
+            	playsinline={true}
+        	/>
+        	<video
+            	bind:this={video_source}
+            	class:hide={!webcam_received}
+            	class:flip={stream_state != "open" ||
+                	(stream_state === "open" && include_audio)}
+            	autoplay={true}
+            	playsinline={true}
+        	/>
+    	</div>
+	 {:else}
+	 	<video
+	 		bind:this={video_source}
+	 		class:hide={!webcam_accessed}
+	 		class:flip={(stream_state != "open") || (stream_state === "open" && include_audio)}
+	 		autoplay={true}
+	 		playsinline={true}
+ 		/>
+	 {/if}
+	
+
 	<!-- svelte-ignore a11y-missing-attribute -->
 	{#if !webcam_accessed}
 		<div
@@ -361,6 +392,12 @@
 		display: none;
 	}
 
+	.video-wrap {
+        display: flex;
+        justify-content: center;
+        align-items: center;
+    }
+
 	video {
 		width: var(--size-full);
 		height: var(--size-full);