using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Net.Http; using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; using JetBrains.Annotations; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using UnityEngine.Networking; using Debug = UnityEngine.Debug; namespace Needle.Cloud.Runtime { internal static class DownloadManager { public struct DownloadOptions { [CanBeNull] public string Password; public CancellationToken? Cancellation; [CanBeNull] public IProgress Progress; [CanBeNull] public string Filename; [CanBeNull] public string TargetDirectory; } public struct DownloadResult { public bool Success; public string Path; public string Error; public bool NeedsPassword; } private static HttpClient client; private static readonly Dictionary> runningDownloads = new Dictionary>(); public static Task Download(string url, DownloadOptions options = default) { if (runningDownloads.TryGetValue(url, out var task)) { if (task != null && !task.IsCompleted) { return task; } } try { var downloadTask = InternalDownload(url, options); runningDownloads[url] = downloadTask; return downloadTask; } catch (ObjectDisposedException) { // Ignore } catch (TaskCanceledException) { // Ignore } return Task.FromResult(new DownloadResult() { Success = false, Path = null, Error = "Download failed" }); } private static async Task InternalDownload(string url, DownloadOptions options) { if (string.IsNullOrEmpty(url) || !url.StartsWith("http")) { return new DownloadResult() { Success = false, Path = null, Error = "Invalid URL" }; } if (new Uri(url).Host.EndsWith("needle.tools") == false) { return new DownloadResult() { Success = false, Path = null, Error = "Invalid URL: Only cloud.needle.tools is allowed" }; } client ??= new HttpClient(); var headMessage = new HttpRequestMessage(HttpMethod.Head, url); headMessage.Headers.Add("User-Agent", "Needle Cloud Unity Client"); if (!string.IsNullOrEmpty(options.Password)) { var encodedPassword = Convert.ToBase64String(Encoding.ASCII.GetBytes($":{options.Password}")); headMessage.Headers.Add("Authorization", $"Basic {encodedPassword}"); } var headResponse = await client.SendAsync(headMessage); if (headResponse.IsSuccessStatusCode) { var headers = headResponse.Headers; var contentHeaders = headResponse.Content.Headers; if (headers.TryGetValues("X-Needle-Protected", out var protectionStrings)) { var passwordProtection = protectionStrings.FirstOrDefault(s => s == "password"); if (passwordProtection != null) { return new DownloadResult() { Success = false, Path = null, Error = "Asset is password protected", NeedsPassword = true }; } } var hasEtag = headers.TryGetValues("ETag", out var etagEnumerable); if (!hasEtag) { return new DownloadResult() { Success = false, Path = null, Error = "Failed to get ETag" }; } var etag = etagEnumerable.FirstOrDefault(); var ext = ".glb"; if (!string.IsNullOrEmpty(options.Filename)) { ext = ""; } else if (contentHeaders.TryGetValues("Content-Type", out var value)) { var contentType = value.FirstOrDefault(); switch (contentType) { case "model/gltf+json": ext = ".gltf"; break; case "model/gltf-binary": ext = ".glb"; break; } } var contentLength = 0; if (contentHeaders.TryGetValues("Content-Length", out var lengthValues)) { if (int.TryParse(lengthValues.FirstOrDefault(), out var length)) { contentLength = length; } } var filename = options.Filename ?? $"asset{ext}"; if (string.IsNullOrEmpty(options.Filename)) { if(contentHeaders.TryGetValues("Content-Disposition", out var dispositionValues)) { var disposition = dispositionValues.FirstOrDefault(); if (!string.IsNullOrEmpty(disposition)) { var parts = disposition.Split(';'); foreach (var part in parts) { if (part.Trim().StartsWith("filename=")) { filename = part.Trim().Substring(9); filename = filename.Trim('"'); // ensure we dont have illegal characters in the filename if (filename.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) { // remove illegal characters filename = new string(filename.Where(c => !Path.GetInvalidFileNameChars().Contains(c)).ToArray()); } if (filename.Length <= 0) { filename = $"asset{ext}"; } break; } } } } } // We hash the etag because the filepath gets too long otherwise var md5 = CreateMD5HashFromEtag(etag); var targetDirectory = options.TargetDirectory ?? $"Library/Needle/Cloud/{md5}"; var targetFilePath = $"{targetDirectory}/{filename}"; // Calculate the directory at the end because the filename *can* also contain a relative path targetDirectory = Path.GetDirectoryName(targetFilePath); if(!string.IsNullOrEmpty(targetDirectory)) Directory.CreateDirectory(targetDirectory); var fileInfo = new FileInfo(targetFilePath); if (!fileInfo.Exists || fileInfo.Length <= 0 || (contentLength > 0 && fileInfo.Length != contentLength)) { var sizeInMBStr = contentLength > 0 ? $" ({((float)contentLength / 1024 / 1024f):0.00} MB)" : ""; Debug.Log($"Downloading asset{sizeInMBStr} from {url}"); // TODO: return meta data about the asset (ID, thumbnail, name). Some of it is in Content-Disposition already var meta = new JObject(); meta["url"] = url; meta["etag"] = etag; meta["content_length"] = contentLength; meta["content_type"] = contentHeaders.ContentType?.MediaType; meta["filename"] = filename; var metaFilePath = targetDirectory + "/.asset.json"; var json = JsonConvert.SerializeObject(meta, Formatting.Indented); File.WriteAllText(metaFilePath, json); // var unityWebRequest = new UnityEngine.Networking.UnityWebRequest(url); // var op = unityWebRequest.SendWebRequest(); // while (op.isDone == false) // { // await Task.Yield(); // Debug.Log(op.progress); // } var downloadMessage = new HttpRequestMessage(HttpMethod.Get, url); downloadMessage.Headers.Add("User-Agent", "Needle Cloud Unity Client"); if(headMessage.Headers.TryGetValues("Authorization", out var auth)) downloadMessage.Headers.Add("Authorization", auth); var downloadResponse = await client.SendAsync(downloadMessage); if (downloadResponse.IsSuccessStatusCode) { options.Progress?.Report(0); var downloadStream = await downloadResponse.Content.ReadAsStreamAsync(); var filestream = File.Create(targetFilePath); await downloadStream.CopyToAsync(filestream); // var relativeProgress = new Progress(totalBytes => options.Progress?.Report((float)totalBytes / contentLength)); // while (!task.IsCompleted) // { // Debug.Log("Downloading..."); // await Task.Yield(); // } filestream.Close(); downloadStream.Close(); options.Progress?.Report(1.0f); Debug.Log($"Downloaded asset to \"{targetFilePath}\""); } else { Debug.LogError($"Failed to download asset from {url}"); } } if (File.Exists(targetFilePath)) { return new DownloadResult() { Success = true, Path = targetFilePath, Error = null }; } } return new DownloadResult() { Success = false, Path = null, Error = $"Failed to access asset: {headResponse.StatusCode}\n{url}" }; } private static string CreateMD5HashFromEtag(string etag) { // Create a md5 hash using var md5 = MD5.Create(); var inputBytes = System.Text.Encoding.ASCII.GetBytes(etag); return BitConverter.ToString(md5.ComputeHash(inputBytes)).Replace("-", "").ToLower(); } private static UnityWebRequestAwaiter GetAwaiter(this UnityWebRequestAsyncOperation asyncOp) { return new UnityWebRequestAwaiter(asyncOp); } } [DebuggerNonUserCode] internal readonly struct UnityWebRequestAwaiter : INotifyCompletion { private readonly UnityWebRequestAsyncOperation _asyncOperation; public bool IsCompleted => _asyncOperation.isDone; public UnityWebRequestAwaiter(UnityWebRequestAsyncOperation asyncOperation) => _asyncOperation = asyncOperation; public void OnCompleted(Action continuation) => _asyncOperation.completed += _ => continuation(); public UnityWebRequest GetResult() => _asyncOperation.webRequest; } }