using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Needle.Engine.Utils; using UnityEngine; using UnityEngine.Networking; using System; #if UNITY_EDITOR using UnityEditor; using UnityGLTF; #endif namespace Needle.Engine { internal static class Tools { public static bool IsUploadingToFTP => uploadingToFTPTask != null && !uploadingToFTPTask.IsCompleted; private static Task uploadingToFTPTask; internal static string UseNpmRegistry = "npm_config_registry=https://registry.npmjs.org "; public static Task UploadToFTP(string server, string username, string password, string localpath, string remotepath, bool sftp, bool delete, int port = -1, CancellationToken cancellationToken = default ) { var version = NpmUnityEditorVersions.TryGetRecommendedVersion("@needle-tools/helper", "latest"); Debug.Log($"Begin FTP upload...\nThis might take a moment if it's the first time. Subsequent uploads will be faster since files that did not change will be skipped. Using version {version}"); var cmd = $"npx --yes @needle-tools/helper@{version}"; cmd += $" upload-ftp"; cmd += $" --server \"{server}\""; cmd += $" --username \"{username}\""; cmd += $" --password \"{password}\""; cmd += $" --localpath \"{localpath}\""; cmd += $" --remotepath \"{remotepath}\""; if (port >= 0) cmd += " --port " + port; if (sftp) cmd += " --sftp"; if (delete) cmd += " --delete"; return uploadingToFTPTask = ProcessHelper.RunCommand(cmd, null, null, true, true, -1, cancellationToken); } public static bool IsCloningRepository => CloneRepositoryTask != null && !CloneRepositoryTask.IsCompleted; private static Task CloneRepositoryTask; public static Task CloneRepository(string url, string targetDirectory) { var version = NpmUnityEditorVersions.TryGetRecommendedVersion("@needle-tools/helper", "latest"); Debug.Log($"Begin cloning repository...\nThis might take a moment if it's the first time. Using version {version}"); var cmd = $"npx --yes @needle-tools/helper@{version}"; cmd += " git-clone"; cmd += " --url \"" + url + "\""; cmd += " --targetDir \"" + targetDirectory + "\""; return CloneRepositoryTask = ProcessHelper.RunCommand(cmd, ""); } public static async Task GenerateFonts(string fontPath, string targetDirectory, string charsetPath) { var version = NpmUnityEditorVersions.TryGetRecommendedVersion("@needle-tools/helper", "latest"); var cmd = $"npx --yes @needle-tools/helper@{version} generate-font-atlas"; cmd += $" --fontPath \"{fontPath}\""; cmd += $" --targetDirectory \"{Path.GetFullPath(targetDirectory)}\""; cmd += $" --charset \"{charsetPath}\""; Debug.Log($"Generating fonts...\nThis might take a moment if it's the first time. Using version {version}\n\nCMD: {cmd}\n\n"); var res = await ProcessHelper.RunCommand(cmd, null, new ProcessHelper.RunOptions() { silent = true }); if (!res.success) { if (res.npmCacheLines?.Count > 0) { await DeleteNpmDirectoriesFromLogs(res.npmCacheLines); await Task.Delay(5000); res = await ProcessHelper.RunCommand(cmd, null, new ProcessHelper.RunOptions()); if (!res.success) { Debug.LogError("Error: NPM cache error: Please try again!\nIf the problem persists, please report a bug via the menu item \"Needle Engine/Report a Bug\" and provide a short description about the error."); } } } return res.success; } public static async Task UploadBugReport(string pathToZipFile, string description) { #if UNITY_EDITOR if (!await HiddenProject.Initialize()) return false; var packagePath = HiddenProject.ToolsPath; var userName = CloudProjectSettings.userName; if (!string.IsNullOrEmpty(CloudProjectSettings.organizationName)) userName += "@" + CloudProjectSettings.organizationName; const int maxLength = 10000; if (description.Length > maxLength) description = description.Substring(0, maxLength); var encodedDescription = UnityWebRequest.EscapeURL(description); var editorVersion = Application.unityVersion; var packageVersion = ProjectInfo.GetCurrentPackageVersion(Constants.UnityPackageName, out _); var cmd = $"npm run tool:upload-bugreport --" + $" --file \"{pathToZipFile}\"" + $" --source \"Unity {editorVersion}, {Constants.UnityPackageName}@{packageVersion}\"" + $" --user \"{userName}\"" + $" --description \"{encodedDescription}\""; var res = await ProcessHelper.RunCommand(cmd, packagePath); if (res) { return true; } return false; #else await Task.Yield(); return false; #endif } internal static string TransformLogFilePath => Application.dataPath + "/../Logs/Needle-gltf-transform.log"; public static async Task Transform(string fileOrDirectory, bool progressive = true, bool compress = true) { var version = NpmUnityEditorVersions.TryGetRecommendedVersion("@needle-tools/gltf-build-pipeline", "latest"); var cmd = $"npx --yes @needle-tools/gltf-build-pipeline@{version} transform \"{fileOrDirectory}\""; cmd += progressive ? " --progressive true" : " --progressive false"; cmd += compress ? " --compress true" : " --compress false"; return await ProcessHelper.RunCommand(cmd, null, TransformLogFilePath); } public static async Task Transform_Compress(string fileOrDirectory, string projectDirectory = null) { if (!Directory.Exists(fileOrDirectory) && !File.Exists(fileOrDirectory)) { Debug.LogError( $"[{nameof(Tools)}.{nameof(Transform_Compress)}] Directory or file not found \"{fileOrDirectory}\", not compressing."); return false; } var version = NpmUnityEditorVersions.TryGetRecommendedVersion("@needle-tools/gltf-build-pipeline", "latest"); var cmd = "npx --yes @needle-tools/gltf-build-pipeline@" + version + " transform \"" + fileOrDirectory + "\" --progressive false"; return await ProcessHelper.RunCommand(cmd, null, TransformLogFilePath); } public static async Task Transform_Progressive(string fileOrDirectory) { var version = NpmUnityEditorVersions.TryGetRecommendedVersion("@needle-tools/gltf-build-pipeline", "latest"); var cmd = "npx --yes @needle-tools/gltf-build-pipeline@" + version + " transform \"" + fileOrDirectory + "\" --compress false"; return await ProcessHelper.RunCommand(cmd, null, TransformLogFilePath); } public static async Task ClearCaches() { var version = NpmUnityEditorVersions.TryGetRecommendedVersion("@needle-tools/gltf-build-pipeline", "latest"); var cmd = "npx --yes @needle-tools/gltf-build-pipeline@" + version + " clear-caches"; await ProcessHelper.RunCommand(cmd, null); } private static string BuildPipelinePath { get { var exportInfo = ExportInfo.Get(); if (exportInfo) { var basePath = $"{Path.GetFullPath(exportInfo.GetProjectDirectory())}/node_modules/"; var path = $"{basePath}/{Constants.GltfBuildPipelineNpmPackageName}"; if (Directory.Exists(path)) { NeedleDebug.Log(TracingScenario.Tools, "Found build pipeline at " + path); return path; } var pnpmDirectory = basePath + "/.pnpm"; if (Directory.Exists(pnpmDirectory)) { var pnpmVersions = Directory.GetDirectories(pnpmDirectory, Constants.GltfBuildPipelineNpmPackageName.Replace("/", "+") + "*", SearchOption.TopDirectoryOnly); foreach (var p in pnpmVersions) { var fullPath = p + "/node_modules/" + Constants.GltfBuildPipelineNpmPackageName; if (Directory.Exists(fullPath)) { NeedleDebug.Log(TracingScenario.Tools, "Found build pipeline in pnpm directory at " + fullPath); return fullPath; } } } } NeedleDebug.Log(TracingScenario.Tools, "Use hidden tools build pipeline at" + HiddenProject.BuildPipelinePath); return HiddenProject.BuildPipelinePath; } } internal static async Task ClearNpxCaches(bool silent = false) { var dir = NpmUtils.NpxCacheDirectory; // var directories = NpmUtils.NpmCacheDirectories; // foreach (var dir in directories) { if (Directory.Exists(dir)) { if(!silent) Debug.Log($"Delete npx cache directory \"{dir}\""); NeedleDebug.Log(TracingScenario.Tools, "Delete npx cache directory at " + dir); try { var res = await FileUtils.DeleteDirectoryRecursive(dir); if (!silent) Debug.Log($"{(res ? "Deleted" : "Failed to delete")} npx cache directory \"{dir}\""); NeedleDebug.Log(TracingScenario.Tools, $"{(res ? "Deleted" : "Failed to delete")} npx cache directory \"{dir}\""); } catch (Exception err) { NeedleDebug.LogException(TracingScenario.Tools, err); } } else { if(!silent) Debug.Log($"NPX cache directory does not exist \"{dir}\""); NeedleDebug.Log(TracingScenario.Tools, "NPX cache directory does not exist at " + dir); } } return true; } private static Dictionary cacheDeleteCount = new Dictionary(); internal static async Task DeleteNpmDirectoriesFromLogs(List logs, bool silent = false) { // https://regex101.com/r/IPSpMd/3 var regex = new Regex(@"(?_npx[/\\](?.+?)[/\\]node_modules[/\\].+?)[/\\]"); foreach(var log in logs) { #if UNITY_EDITOR_WIN if (log.Contains("npm-cache")) #else if (log.Contains(".npm")) #endif { var match = regex.Match(log); if (match.Success) { var hash = match.Groups["package_hash"]?.Value; if (!string.IsNullOrWhiteSpace(hash) && (cacheDeleteCount.ContainsKey(hash) == false || (cacheDeleteCount.TryGetValue(hash, out var count) && count < 2))) { var fullPackageDir = $"{NpmUtils.NpmCacheDirectory}/_npx/{match.Groups["package_hash"]?.Value}"; if (Directory.Exists(fullPackageDir)) { cacheDeleteCount[hash] = cacheDeleteCount.TryGetValue(hash, out var value) ? value + 1 : 1; // delete the whole _npx/package_hash directory e.g. in \npm-cache\_npx\56f1b756933f83c0 NeedleDebug.Log(TracingScenario.Tools, $"Deleting corrupted npm cache directory at {fullPackageDir}"); if(!silent) Debug.Log($"Deleting corrupted npm cache directory at {fullPackageDir}".LowContrast()); var res = await FileUtils.DeleteDirectoryRecursive(fullPackageDir); if (res) NeedleDebug.Log(TracingScenario.Tools, $"Successfully corrupted npm cache directory at {fullPackageDir}"); else NeedleDebug.LogError(TracingScenario.Tools, $"Failed to delete corrupted npm cache directory at {fullPackageDir}"); continue; } } // Fallback to deleting the single package directory inside node_modules var directory = match.Groups["dir"]?.Value; var fullPath = $"{NpmUtils.NpmCacheDirectory}/{directory}"; if (directory != null && Directory.Exists(fullPath)) { NeedleDebug.Log(TracingScenario.Tools, $"Deleting corrupted npm cache directory at {fullPath}"); if(!silent) Debug.Log($"Deleting corrupted npm cache directory at {fullPath}".LowContrast()); var res = await FileUtils.DeleteDirectoryRecursive(fullPath); if (res) NeedleDebug.Log(TracingScenario.Tools, $"Successfully corrupted npm cache directory at {fullPath}"); else NeedleDebug.LogError(TracingScenario.Tools, $"Failed to delete corrupted npm cache directory at {fullPath}"); continue; } } } NeedleDebug.Log(TracingScenario.Tools, $"Could not find npm cache directory in log: \"{log}\""); } } } }