diff --git a/source/app.d b/source/app.d index 80540d3..7ecb83a 100644 --- a/source/app.d +++ b/source/app.d @@ -11,14 +11,14 @@ import dub.dependency; import dub.dub; import dub.generators.generator; +import dub.internal.std.process; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.inet.url; import dub.package_; import dub.project; import dub.registry; -import vibecompat.core.file; -import vibecompat.core.log; -import vibecompat.inet.url; - import std.algorithm; import std.array; import std.conv; @@ -26,7 +26,6 @@ import std.exception; import std.file; import std.getopt; -import stdx.process; int main(string[] args) diff --git a/source/dub/compilers/compiler.d b/source/dub/compilers/compiler.d index f277dfb..09de7e3 100644 --- a/source/dub/compilers/compiler.d +++ b/source/dub/compilers/compiler.d @@ -10,12 +10,12 @@ import dub.compilers.dmd; import dub.compilers.gdc; import dub.compilers.ldc; +import dub.internal.vibecompat.data.json; +import dub.internal.vibecompat.inet.path; import std.algorithm; import std.array; import std.exception; -import vibecompat.data.json; -import vibecompat.inet.path; static this() @@ -47,7 +47,8 @@ void warnOnSpecialCompilerFlags(string[] compiler_flags, string package_name, string config_name) { - import vibecompat.core.log; + import dub.internal.vibecompat.core.log; + struct SpecialFlag { string[] flags; string alternative; diff --git a/source/dub/compilers/dmd.d b/source/dub/compilers/dmd.d index 1379cd1..cc1a1d3 100644 --- a/source/dub/compilers/dmd.d +++ b/source/dub/compilers/dmd.d @@ -8,15 +8,15 @@ module dub.compilers.dmd; import dub.compilers.compiler; +import dub.internal.std.process; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.inet.path; import dub.platform; import std.algorithm; import std.array; import std.conv; import std.exception; -import stdx.process; -import vibecompat.core.log; -import vibecompat.inet.path; class DmdCompiler : Compiler { diff --git a/source/dub/compilers/gdc.d b/source/dub/compilers/gdc.d index 5a71515..56e675e 100644 --- a/source/dub/compilers/gdc.d +++ b/source/dub/compilers/gdc.d @@ -8,15 +8,15 @@ module dub.compilers.gdc; import dub.compilers.compiler; +import dub.internal.std.process; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.inet.path; import dub.platform; import std.algorithm; import std.array; import std.conv; import std.exception; -import stdx.process; -import vibecompat.core.log; -import vibecompat.inet.path; class GdcCompiler : Compiler { diff --git a/source/dub/compilers/ldc.d b/source/dub/compilers/ldc.d index 3fecfa8..83ea5fd 100644 --- a/source/dub/compilers/ldc.d +++ b/source/dub/compilers/ldc.d @@ -8,15 +8,15 @@ module dub.compilers.ldc; import dub.compilers.compiler; +import dub.internal.std.process; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.inet.path; import dub.platform; import std.algorithm; import std.array; import std.conv; import std.exception; -import stdx.process; -import vibecompat.core.log; -import vibecompat.inet.path; class LdcCompiler : Compiler { diff --git a/source/dub/dependency.d b/source/dub/dependency.d index eb66f0c..e34cff9 100644 --- a/source/dub/dependency.d +++ b/source/dub/dependency.d @@ -7,13 +7,12 @@ */ module dub.dependency; -import dub.utils; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.data.json; +import dub.internal.vibecompat.inet.url; import dub.package_; - -import vibecompat.core.log; -import vibecompat.core.file; -import vibecompat.data.json; -import vibecompat.inet.url; +import dub.utils; import std.array; import std.string; diff --git a/source/dub/dub.d b/source/dub/dub.d index 14173ac..c02a117 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -10,6 +10,11 @@ import dub.compilers.compiler; import dub.dependency; import dub.installation; +import dub.internal.std.process; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.data.json; +import dub.internal.vibecompat.inet.url; import dub.utils; import dub.registry; import dub.package_; @@ -18,10 +23,6 @@ import dub.project; import dub.generators.generator; -import vibecompat.core.file; -import vibecompat.core.log; -import vibecompat.data.json; -import vibecompat.inet.url; // todo: cleanup imports. import std.algorithm; @@ -33,7 +34,6 @@ import std.string; import std.typecons; import std.zip; -import stdx.process; diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index 4a40fd9..b433fb3 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -9,6 +9,10 @@ import dub.compilers.compiler; import dub.generators.generator; +import dub.internal.std.process; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.inet.path; import dub.package_; import dub.packagemanager; import dub.project; @@ -20,11 +24,6 @@ import std.exception; import std.file; import std.string; -import stdx.process; - -import vibecompat.core.file; -import vibecompat.core.log; -import vibecompat.inet.path; class BuildGenerator : ProjectGenerator { diff --git a/source/dub/generators/generator.d b/source/dub/generators/generator.d index 5eb687a..4bccc74 100644 --- a/source/dub/generators/generator.d +++ b/source/dub/generators/generator.d @@ -12,6 +12,9 @@ import dub.generators.monod; import dub.generators.rdmd; import dub.generators.visuald; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.inet.path; import dub.package_; import dub.packagemanager; import dub.project; @@ -19,9 +22,6 @@ import std.exception; import std.file; import std.string; -import vibecompat.core.file; -import vibecompat.core.log; -import vibecompat.inet.path; /** @@ -128,7 +128,7 @@ void runBuildCommands(string[] commands, in BuildSettings build_settings) { - import stdx.process; + import dub.internal.std.process; import dub.utils; string[string] env = environment.toAA(); diff --git a/source/dub/generators/monod.d b/source/dub/generators/monod.d index 439172d..7cf80d5 100644 --- a/source/dub/generators/monod.d +++ b/source/dub/generators/monod.d @@ -9,6 +9,8 @@ import dub.compilers.compiler; import dub.generators.generator; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.log; import dub.package_; import dub.packagemanager; import dub.project; @@ -21,9 +23,6 @@ import std.uuid; import std.exception; -import vibecompat.core.file; -import vibecompat.core.log; - // TODO: handle pre/post build commands diff --git a/source/dub/generators/rdmd.d b/source/dub/generators/rdmd.d index 0bf6b07..cd7ff2e 100644 --- a/source/dub/generators/rdmd.d +++ b/source/dub/generators/rdmd.d @@ -9,6 +9,10 @@ import dub.compilers.compiler; import dub.generators.generator; +import dub.internal.std.process; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.inet.path; import dub.package_; import dub.packagemanager; import dub.project; @@ -20,11 +24,6 @@ import std.exception; import std.file; import std.string; -import stdx.process; - -import vibecompat.core.file; -import vibecompat.core.log; -import vibecompat.inet.path; class RdmdGenerator : ProjectGenerator { diff --git a/source/dub/generators/visuald.d b/source/dub/generators/visuald.d index d618158..21c0f2f 100644 --- a/source/dub/generators/visuald.d +++ b/source/dub/generators/visuald.d @@ -9,6 +9,8 @@ import dub.compilers.compiler; import dub.generators.generator; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.log; import dub.package_; import dub.packagemanager; import dub.project; @@ -21,8 +23,6 @@ import std.uuid; import std.exception; -import vibecompat.core.file; -import vibecompat.core.log; version = VISUALD_SEPERATE_PROJECT_FILES; //version = VISUALD_SINGLE_PROJECT_FILE; diff --git a/source/dub/installation.d b/source/dub/installation.d index 97b94ec..a1d0ca8 100644 --- a/source/dub/installation.d +++ b/source/dub/installation.d Binary files differ diff --git a/source/dub/internal/std/process.d b/source/dub/internal/std/process.d new file mode 100644 index 0000000..0ca1f32 --- /dev/null +++ b/source/dub/internal/std/process.d @@ -0,0 +1,2473 @@ +// Written in the D programming language. + +/** +Functions for starting and interacting with other processes, and for +working with the current process' execution environment. + +Process_handling: +$(UL $(LI + $(LREF spawnProcess) spawns a new _process, optionally assigning it an + arbitrary set of standard input, output, and error streams. + The function returns immediately, leaving the child _process to execute + in parallel with its parent. All other functions in this module that + spawn processes are built around $(D spawnProcess).) +$(LI + $(LREF wait) makes the parent _process wait for a child _process to + terminate. In general one should always do this, to avoid + child _processes becoming "zombies" when the parent _process exits. + Scope guards are perfect for this – see the $(LREF spawnProcess) + documentation for examples.) +$(LI + $(LREF pipeProcess) also spawns a child _process which runs + in parallel with its parent. However, instead of taking + arbitrary streams, it automatically creates a set of + pipes that allow the parent to communicate with the child + through the child's standard input, output, and/or error streams. + This function corresponds roughly to C's $(D popen) function.) +$(LI + $(LREF execute) starts a new _process and waits for it + to complete before returning. Additionally, it captures + the _process' standard output and error streams and returns + the output of these as a string.) +$(LI + $(LREF spawnShell), $(LREF pipeShell) and $(LREF shell) work like + $(D spawnProcess), $(D pipeProcess) and $(D execute), respectively, + except that they take a single command string and run it through + the current user's default command interpreter. + $(D shell) corresponds roughly to C's $(D system) function.) +$(LI + $(LREF kill) attempts to terminate a running process.) +) +Unless the directory of the executable file is explicitly specified, all +functions will search for it in the directories specified in the PATH +environment variable. + +Other_functionality: +$(UL +$(LI + $(LREF pipe) is used to create unidirectional pipes.) +$(LI + $(LREF environment) is an interface through which the current process' + environment variables can be read and manipulated.) +) + +Authors: + $(LINK2 https://github.com/kyllingstad, Lars Tandle Kyllingstad), + $(LINK2 https://github.com/schveiguy, Steven Schveighoffer), + $(LINK2 https://github.com/cybershadow, Vladimir Panteleev) +Copyright: + Copyright (c) 2013, the authors. All rights reserved. +Source: + $(PHOBOSSRC std/_process.d) +Macros: + WIKI=Phobos/StdProcess + OBJECTREF=$(D $(LINK2 object.html#$0,$0)) +*/ +module dub.internal.std.process; + +version (Posix) +{ + import core.stdc.errno; + import core.stdc.string; + import core.sys.posix.stdio; + import core.sys.posix.unistd; + import core.sys.posix.sys.wait; +} +version (Windows) +{ + import core.stdc.stdio; + import core.sys.windows.windows; + import std.utf; + import std.windows.syserror; +} +import std.algorithm; +import std.array; +import std.conv; +import std.exception; +import std.path; +import std.stdio; +import std.string; +import std.typecons; + + +// When the DMC runtime is used, we have to use some custom functions +// to convert between Windows file handles and FILE*s. +version (Win32) version (DigitalMars) version = DMC_RUNTIME; + + +// Some of the following should be moved to druntime. +private: + +// Windows API declarations. +version (Windows) +{ + extern(Windows) BOOL GetHandleInformation(HANDLE hObject, + LPDWORD lpdwFlags); + extern(Windows) BOOL SetHandleInformation(HANDLE hObject, + DWORD dwMask, + DWORD dwFlags); + extern(Windows) BOOL TerminateProcess(HANDLE hProcess, + UINT uExitCode); + extern(Windows) LPWSTR* CommandLineToArgvW(LPCWSTR lpCmdLine, + int* pNumArgs); + enum + { + HANDLE_FLAG_INHERIT = 0x1, + HANDLE_FLAG_PROTECT_FROM_CLOSE = 0x2, + } + enum CREATE_UNICODE_ENVIRONMENT = 0x400; +} + +// Microsoft Visual C Runtime (MSVCRT) declarations. +version (Windows) +{ + version (DMC_RUNTIME) { } else + { + import core.stdc.stdint; + extern(C) + { + int _fileno(FILE* stream); + HANDLE _get_osfhandle(int fd); + int _open_osfhandle(HANDLE osfhandle, int flags); + FILE* _fdopen(int fd, const (char)* mode); + int _close(int fd); + } + enum + { + STDIN_FILENO = 0, + STDOUT_FILENO = 1, + STDERR_FILENO = 2, + } + enum + { + _O_RDONLY = 0x0000, + _O_APPEND = 0x0004, + _O_TEXT = 0x4000, + } + } +} + +// POSIX API declarations. +version (Posix) +{ + version (OSX) + { + // https://www.gnu.org/software/gnulib/manual/html_node/environ.html + extern(C) char*** _NSGetEnviron(); + __gshared const char** environ; + shared static this() { environ = *_NSGetEnviron(); } + } + else + { + // Made available by the C runtime: + extern(C) extern __gshared const char** environ; + } +} + + +// Actual module classes/functions start here. +public: + + +// ============================================================================= +// Functions and classes for process management. +// ============================================================================= + + +/** +Spawns a new _process, optionally assigning it an +arbitrary set of standard input, output, and error streams. +The function returns immediately, leaving the child _process to execute +in parallel with its parent. + +Command_line: +There are four overloads of this function. The first two take an array +of strings, $(D args), which should contain the program name as the +zeroth element and any command-line arguments in subsequent elements. +The third and fourth versions are included for convenience, and may be +used when there are no command-line arguments. They take a single string, +$(D program), which specifies the program name. + +Unless a directory is specified in $(D args[0]) or $(D program), +$(D spawnProcess) will search for the program in the directories listed +in the PATH environment variable. To run an executable in the current +directory, use $(D "./$(I executable_name)"). +--- +// Run an executable called "prog" located in the current working +// directory: +auto pid = spawnProcess("./prog"); +scope(exit) wait(pid); +// We can do something else while the program runs. The scope guard +// ensures that the process is waited for at the end of the scope. +... + +// Run DMD on the file "myprog.d", specifying a few compiler switches: +auto dmdPid = spawnProcess(["dmd", "-O", "-release", "-inline", "myprog.d" ]); +if (wait(dmdPid) != 0) + writeln("Compilation failed!"); +--- + +Environment_variables: +With the first and third $(D spawnProcess) overloads, one can specify +the environment variables of the child process using the $(D environmentVars) +parameter. With the second and fourth overload, the child process inherits +its parent's environment variables. + +To make the child inherit the parent's environment $(I plus) one or more +additional variables, first use $(D $(LREF environment).$(LREF toAA)) to +obtain an associative array that contains the parent's environment +variables, and add the new variables to it before passing it to +$(D spawnProcess). +--- +auto envVars = environment.toAA(); +envVars["FOO"] = "bar"; +wait(spawnProcess("prog", envVars)); +--- + +Standard_streams: +The optional arguments $(D stdin_), $(D stdout_) and $(D stderr_) may +be used to assign arbitrary $(XREF stdio,File) objects as the standard +input, output and error streams, respectively, of the child process. The +former must be opened for reading, while the latter two must be opened for +writing. The default is for the child process to inherit the standard +streams of its parent. +--- +// Run DMD on the file myprog.d, logging any error messages to a +// file named errors.log. +auto logFile = File("errors.log", "w"); +auto pid = spawnProcess(["dmd", "myprog.d"], + std.stdio.stdin, + std.stdio.stdout, + logFile); +if (wait(pid) != 0) + writeln("Compilation failed. See errors.log for details."); +--- + +Note that if you pass a $(D File) object that is $(I not) +one of the standard input/output/error streams of the parent process, +that stream will by default be $(I closed) in the parent process when +this function returns. See the $(LREF Config) documentation below for +information about how to disable this behaviour. + +Beware of buffering issues when passing $(D File) objects to +$(D spawnProcess). The child process will inherit the low-level raw +read/write offset associated with the underlying file descriptor, but +it will not be aware of any buffered data. In cases where this matters +(e.g. when a file should be aligned before being passed on to the +child process), it may be a good idea to use unbuffered streams, or at +least ensure all relevant buffers are flushed. + +Params: +args = An array which contains the program name as the first element + and any command-line arguments in the following elements. +program = The program name, $(I without) command-line arguments. +environmentVars = The environment variables for the child process may + be specified using this parameter. By default it is $(D null), + which means that, the child process inherits the environment of + the parent process. +stdin_ = The standard input stream of the child process. + This can be any $(XREF stdio,File) that is opened for reading. + By default the child process inherits the parent's input + stream. +stdout_ = The standard output stream of the child process. + This can be any $(XREF stdio,File) that is opened for writing. + By default the child process inherits the parent's output + stream. +stderr_ = The standard error stream of the child process. + This can be any $(XREF stdio,File) that is opened for writing. + By default the child process inherits the parent's error + stream. +config = Options that control the behaviour of $(D spawnProcess). + See the $(LREF Config) documentation for details. + +Returns: +A $(LREF Pid) object that corresponds to the spawned process. + +Throws: +$(LREF ProcessException) on failure to start the process.$(BR) +$(XREF stdio,StdioException) on failure to pass one of the streams + to the child process (Windows only).$(BR) +$(CXREF exception,RangeError) if $(D args) is empty. +*/ +Pid spawnProcess(in char[][] args, + const string[string] environmentVars, + File stdin_ = std.stdio.stdin, + File stdout_ = std.stdio.stdout, + File stderr_ = std.stdio.stderr, + Config config = Config.none) + @trusted // TODO: Should be @safe +{ + version (Windows) auto args2 = escapeShellArguments(args); + else version (Posix) alias args2 = args; + return spawnProcessImpl(args2, toEnvz(environmentVars), + stdin_, stdout_, stderr_, config); +} + +/// ditto +Pid spawnProcess(in char[][] args, + File stdin_ = std.stdio.stdin, + File stdout_ = std.stdio.stdout, + File stderr_ = std.stdio.stderr, + Config config = Config.none) + @trusted // TODO: Should be @safe +{ + version (Windows) auto args2 = escapeShellArguments(args); + else version (Posix) alias args2 = args; + return spawnProcessImpl(args2, null, stdin_, stdout_, stderr_, config); +} + +/// ditto +Pid spawnProcess(in char[] program, + const string[string] environmentVars, + File stdin_ = std.stdio.stdin, + File stdout_ = std.stdio.stdout, + File stderr_ = std.stdio.stderr, + Config config = Config.none) + @trusted +{ + return spawnProcess((&program)[0 .. 1], environmentVars, + stdin_, stdout_, stderr_, config); +} + +/// ditto +Pid spawnProcess(in char[] program, + File stdin_ = std.stdio.stdin, + File stdout_ = std.stdio.stdout, + File stderr_ = std.stdio.stderr, + Config config = Config.none) + @trusted +{ + return spawnProcess((&program)[0 .. 1], + stdin_, stdout_, stderr_, config); +} + +/* +Implementation of spawnProcess() for POSIX. + +envz should be a zero-terminated array of zero-terminated strings +on the form "var=value". +*/ +version (Posix) +private Pid spawnProcessImpl(in char[][] args, + const(char*)* envz, + File stdin_, + File stdout_, + File stderr_, + Config config) + @trusted // TODO: Should be @safe +{ + const(char)[] name = args[0]; + if (any!isDirSeparator(name)) + { + if (!isExecutable(name)) + throw new ProcessException(text("Not an executable file: ", name)); + } + else + { + name = searchPathFor(name); + if (name is null) + throw new ProcessException(text("Executable file not found: ", name)); + } + + // Convert program name and arguments to C-style strings. + auto argz = new const(char)*[args.length+1]; + argz[0] = toStringz(name); + foreach (i; 1 .. args.length) argz[i] = toStringz(args[i]); + argz[$-1] = null; + + // Use parent's environment variables? + if (envz is null) envz = environ; + + // Get the file descriptors of the streams. + // These could potentially be invalid, but that is OK. If so, later calls + // to dup2() and close() will just silently fail without causing any harm. + auto stdinFD = core.stdc.stdio.fileno(stdin_.getFP()); + auto stdoutFD = core.stdc.stdio.fileno(stdout_.getFP()); + auto stderrFD = core.stdc.stdio.fileno(stderr_.getFP()); + + auto id = fork(); + if (id < 0) + throw ProcessException.newFromErrno("Failed to spawn new process"); + if (id == 0) + { + // Child process + + // Redirect streams and close the old file descriptors. + // In the case that stderr is redirected to stdout, we need + // to backup the file descriptor since stdout may be redirected + // as well. + if (stderrFD == STDOUT_FILENO) stderrFD = dup(stderrFD); + dup2(stdinFD, STDIN_FILENO); + dup2(stdoutFD, STDOUT_FILENO); + dup2(stderrFD, STDERR_FILENO); + + // Close the old file descriptors, unless they are + // either of the standard streams. + if (stdinFD > STDERR_FILENO) close(stdinFD); + if (stdoutFD > STDERR_FILENO) close(stdoutFD); + if (stderrFD > STDERR_FILENO) close(stderrFD); + + // Execute program. + execve(argz[0], argz.ptr, envz); + + // If execution fails, exit as quickly as possible. + perror("spawnProcess(): Failed to execute program"); + _exit(1); + assert (0); + } + else + { + // Parent process: Close streams and return. + if (stdinFD > STDERR_FILENO && !(config & Config.noCloseStdin)) + stdin_.close(); + if (stdoutFD > STDERR_FILENO && !(config & Config.noCloseStdout)) + stdout_.close(); + if (stderrFD > STDERR_FILENO && !(config & Config.noCloseStderr)) + stderr_.close(); + return new Pid(id); + } +} + +/* +Implementation of spawnProcess() for Windows. + +commandLine must contain the entire command line, properly +quoted/escaped as required by CreateProcessW(). + +envz must be a pointer to a block of UTF-16 characters on the form +"var1=value1\0var2=value2\0...varN=valueN\0\0". +*/ +version (Windows) +private Pid spawnProcessImpl(in char[] commandLine, + LPVOID envz, + File stdin_, + File stdout_, + File stderr_, + Config config) + @trusted +{ + auto commandz = toUTFz!(wchar*)(commandLine); + + // Startup info for CreateProcessW(). + STARTUPINFO_W startinfo; + startinfo.cb = startinfo.sizeof; + startinfo.dwFlags = STARTF_USESTDHANDLES; + + // Extract file descriptors and HANDLEs from the streams and make the + // handles inheritable. + static void prepareStream(ref File file, DWORD stdHandle, string which, + out int fileDescriptor, out HANDLE handle) + { + fileDescriptor = _fileno(file.getFP()); + if (fileDescriptor < 0) handle = GetStdHandle(stdHandle); + else + { + version (DMC_RUNTIME) handle = _fdToHandle(fileDescriptor); + else /* MSVCRT */ handle = _get_osfhandle(fileDescriptor); + } + DWORD dwFlags; + GetHandleInformation(handle, &dwFlags); + if (!(dwFlags & HANDLE_FLAG_INHERIT)) + { + if (!SetHandleInformation(handle, + HANDLE_FLAG_INHERIT, + HANDLE_FLAG_INHERIT)) + { + throw new StdioException( + "Failed to make "~which~" stream inheritable by child process (" + ~sysErrorString(GetLastError()) ~ ')', + 0); + } + } + } + int stdinFD = -1, stdoutFD = -1, stderrFD = -1; + prepareStream(stdin_, STD_INPUT_HANDLE, "stdin" , stdinFD, startinfo.hStdInput ); + prepareStream(stdout_, STD_OUTPUT_HANDLE, "stdout", stdoutFD, startinfo.hStdOutput); + prepareStream(stderr_, STD_ERROR_HANDLE, "stderr", stderrFD, startinfo.hStdError ); + + // Create process. + PROCESS_INFORMATION pi; + DWORD dwCreationFlags = CREATE_UNICODE_ENVIRONMENT | + ((config & Config.gui) ? CREATE_NO_WINDOW : 0); + if (!CreateProcessW(null, commandz, null, null, true, dwCreationFlags, + envz, null, &startinfo, &pi)) + throw ProcessException.newFromLastError("Failed to spawn new process"); + + // figure out if we should close any of the streams + if (stdinFD > STDERR_FILENO && !(config & Config.noCloseStdin)) + stdin_.close(); + if (stdoutFD > STDERR_FILENO && !(config & Config.noCloseStdout)) + stdout_.close(); + if (stderrFD > STDERR_FILENO && !(config & Config.noCloseStderr)) + stderr_.close(); + + // close the thread handle in the process info structure + CloseHandle(pi.hThread); + + return new Pid(pi.dwProcessId, pi.hProcess); +} + +// Searches the PATH variable for the given executable file, +// (checking that it is in fact executable). +version (Posix) +private string searchPathFor(in char[] executable) + @trusted //TODO: @safe nothrow +{ + auto pathz = core.stdc.stdlib.getenv("PATH"); + if (pathz == null) return null; + + foreach (dir; splitter(to!string(pathz), ':')) + { + auto execPath = buildPath(dir, executable); + if (isExecutable(execPath)) return execPath; + } + + return null; +} + +// Converts a string[string] array to a C array of C strings +// on the form "key=value". +version (Posix) +private const(char)** toEnvz(const string[string] env) + @trusted //TODO: @safe pure nothrow +{ + alias const(char)* stringz_t; + auto envz = new stringz_t[](env.length+1); + int i = 0; + foreach (k, v; env) envz[i++] = (k~'='~v~'\0').ptr; + envz[i] = null; + return envz.ptr; +} + +// Converts a string[string] array to a block of 16-bit +// characters on the form "key=value\0key=value\0...\0\0" +version (Windows) +private LPVOID toEnvz(const string[string] env) + @trusted //TODO: @safe pure nothrow +{ + auto envz = appender!(wchar[])(); + foreach(k, v; env) + { + envz.put(k); + envz.put('='); + envz.put(v); + envz.put('\0'); + } + envz.put('\0'); + return envz.data.ptr; +} + +// Checks whether the file exists and can be executed by the +// current user. +version (Posix) +private bool isExecutable(in char[] path) @trusted //TODO: @safe nothrow +{ + return (access(toStringz(path), X_OK) == 0); +} + +unittest +{ + TestScript prog1 = "exit 0"; + assert (wait(spawnProcess(prog1.path)) == 0); + + TestScript prog2 = "exit 123"; + auto pid2 = spawnProcess([prog2.path]); + assert (wait(pid2) == 123); + assert (wait(pid2) == 123); // Exit code is cached. + + version (Windows) TestScript prog3 = + "if not -%1-==-foo- ( exit 1 ) + if not -%2-==-bar- ( exit 1 ) + exit 0"; + else version (Posix) TestScript prog3 = + `if test "$1" != "foo"; then exit 1; fi + if test "$2" != "bar"; then exit 1; fi + exit 0`; + assert (wait(spawnProcess([ prog3.path, "foo", "bar"])) == 0); + assert (wait(spawnProcess(prog3.path)) == 1); + + version (Windows) TestScript prog4 = + "if %hello%==world ( exit 0 ) + exit 1"; + version (Posix) TestScript prog4 = + "if test $hello = world; then exit 0; fi + exit 1"; + auto env = [ "hello" : "world" ]; + assert (wait(spawnProcess(prog4.path, env)) == 0); + assert (wait(spawnProcess([prog4.path], env)) == 0); + + version (Windows) TestScript prog5 = + "set /p INPUT= + echo %INPUT% output %1 + echo %INPUT% error %2 1>&2"; + else version (Posix) TestScript prog5 = + "read INPUT + echo $INPUT output $1 + echo $INPUT error $2 >&2"; + auto pipe5i = pipe(); + auto pipe5o = pipe(); + auto pipe5e = pipe(); + auto pid5 = spawnProcess([ prog5.path, "foo", "bar" ], + pipe5i.readEnd, pipe5o.writeEnd, pipe5e.writeEnd); + pipe5i.writeEnd.writeln("input"); + pipe5i.writeEnd.flush(); + assert (pipe5o.readEnd.readln().chomp() == "input output foo"); + assert (pipe5e.readEnd.readln().chomp().stripRight() == "input error bar"); + wait(pid5); + + import std.ascii, std.file, std.uuid; + auto path6i = buildPath(tempDir(), randomUUID().toString()); + auto path6o = buildPath(tempDir(), randomUUID().toString()); + auto path6e = buildPath(tempDir(), randomUUID().toString()); + std.file.write(path6i, "INPUT"~std.ascii.newline); + auto file6i = File(path6i, "r"); + auto file6o = File(path6o, "w"); + auto file6e = File(path6e, "w"); + auto pid6 = spawnProcess([prog5.path, "bar", "baz" ], + file6i, file6o, file6e); + wait(pid6); + assert (readText(path6o).chomp() == "INPUT output bar"); + assert (readText(path6e).chomp().stripRight() == "INPUT error baz"); + remove(path6i); + remove(path6o); + remove(path6e); +} + + +/** +A variation on $(LREF spawnProcess) that runs the given _command through +the current user's preferred _command interpreter (aka. shell). + +The string $(D command) is passed verbatim to the shell, and is therefore +subject to its rules about _command structure, argument/filename quoting +and escaping of special characters. +The path to the shell executable is determined by the $(LREF userShell) +function. + +In all other respects this function works just like $(D spawnProcess). +Please refer to the $(LREF spawnProcess) documentation for descriptions +of the other function parameters, the return value and any exceptions +that may be thrown. +--- +// Run the command/program "foo" on the file named "my file.txt", and +// redirect its output into foo.log. +auto pid = spawnShell(`foo "my file.txt" > foo.log`); +wait(pid); +--- + +See_also: +$(LREF escapeShellCommand), which may be helpful in constructing a +properly quoted and escaped shell command line for the current plattform, +from an array of separate arguments. +*/ +Pid spawnShell(in char[] command, + const string[string] environmentVars, + File stdin_ = std.stdio.stdin, + File stdout_ = std.stdio.stdout, + File stderr_ = std.stdio.stderr, + Config config = Config.none) + @trusted // TODO: Should be @safe +{ + return spawnShellImpl(command, toEnvz(environmentVars), + stdin_, stdout_, stderr_, config); +} + +/// ditto +Pid spawnShell(in char[] command, + File stdin_ = std.stdio.stdin, + File stdout_ = std.stdio.stdout, + File stderr_ = std.stdio.stderr, + Config config = Config.none) + @trusted // TODO: Should be @safe +{ + return spawnShellImpl(command, null, stdin_, stdout_, stderr_, config); +} + +// Implementation of spawnShell() for Windows. +version(Windows) +private Pid spawnShellImpl(in char[] command, + LPVOID envz, + File stdin_ = std.stdio.stdin, + File stdout_ = std.stdio.stdout, + File stderr_ = std.stdio.stderr, + Config config = Config.none) + @trusted // TODO: Should be @safe +{ + auto scmd = escapeShellArguments(userShell, shellSwitch) ~ " " ~ command; + return spawnProcessImpl(scmd, envz, stdin_, stdout_, stderr_, config); +} + +// Implementation of spawnShell() for POSIX. +version(Posix) +private Pid spawnShellImpl(in char[] command, + const char** envz, + File stdin_ = std.stdio.stdin, + File stdout_ = std.stdio.stdout, + File stderr_ = std.stdio.stderr, + Config config = Config.none) + @trusted // TODO: Should be @safe +{ + const(char)[][3] args; + args[0] = userShell; + args[1] = shellSwitch; + args[2] = command; + return spawnProcessImpl(args, envz, stdin_, stdout_, stderr_, config); +} + + + +/** +Flags that control the behaviour of $(LREF spawnProcess) and +$(LREF spawnShell). + +Use bitwise OR to combine flags. + +Example: +--- +auto logFile = File("myapp_error.log", "w"); + +// Start program in a console window (Windows only), redirect +// its error stream to logFile, and leave logFile open in the +// parent process as well. +auto pid = spawnProcess("myapp", stdin, stdout, logFile, + Config.noCloseStderr | Config.gui); +scope(exit) +{ + auto exitCode = wait(pid); + logFile.writeln("myapp exited with code ", exitCode); + logFile.close(); +} +--- +*/ +enum Config +{ + none = 0, + + /** + Unless the child process inherits the standard + input/output/error streams of its parent, one almost + always wants the streams closed in the parent when + $(LREF spawnProcess) returns. Therefore, by default, this + is done. If this is not desirable, pass any of these + options to spawnProcess. + */ + noCloseStdin = 1, + noCloseStdout = 2, /// ditto + noCloseStderr = 4, /// ditto + + /** + On Windows, the child process will by default be run in + a console window. This option wil cause it to run in "GUI mode" + instead, i.e., without a console. On POSIX, it has no effect. + */ + gui = 8, +} + + +/// A handle that corresponds to a spawned process. +final class Pid +{ + /** + The process ID number. + + This is a number that uniquely identifies the process on the operating + system, for at least as long as the process is running. Once $(LREF wait) + has been called on the $(LREF Pid), this method will return an + invalid process ID. + */ + @property int processID() const @safe pure nothrow + { + return _processID; + } + + /** + An operating system handle to the process. + + This handle is used to specify the process in OS-specific APIs. + On POSIX, this function returns a $(D core.sys.posix.sys.types.pid_t) + with the same value as $(LREF processID), while on Windows it returns + a $(D core.sys.windows.windows.HANDLE). + + Once $(LREF wait) has been called on the $(LREF Pid), this method + will return an invalid handle. + */ + // Note: Since HANDLE is a reference, this function cannot be const. + version (Windows) + @property HANDLE osHandle() @safe pure nothrow + { + return _handle; + } + else version (Posix) + @property pid_t osHandle() @safe pure nothrow + { + return _processID; + } + +private: + /* + Pid.performWait() does the dirty work for wait() and nonBlockingWait(). + + If block == true, this function blocks until the process terminates, + sets _processID to terminated, and returns the exit code or terminating + signal as described in the wait() documentation. + + If block == false, this function returns immediately, regardless + of the status of the process. If the process has terminated, the + function has the exact same effect as the blocking version. If not, + it returns 0 and does not modify _processID. + */ + version (Posix) + int performWait(bool block) @trusted + { + if (_processID == terminated) return _exitCode; + int exitCode; + while(true) + { + int status; + auto check = waitpid(_processID, &status, block ? 0 : WNOHANG); + if (check == -1) + { + if (errno == ECHILD) + { + throw new ProcessException( + "Process does not exist or is not a child process."); + } + else + { + // waitpid() was interrupted by a signal. We simply + // restart it. + assert (errno == EINTR); + continue; + } + } + if (!block && check == 0) return 0; + if (WIFEXITED(status)) + { + exitCode = WEXITSTATUS(status); + break; + } + else if (WIFSIGNALED(status)) + { + exitCode = -WTERMSIG(status); + break; + } + // We check again whether the call should be blocking, + // since we don't care about other status changes besides + // "exited" and "terminated by signal". + if (!block) return 0; + + // Process has stopped, but not terminated, so we continue waiting. + } + // Mark Pid as terminated, and cache and return exit code. + _processID = terminated; + _exitCode = exitCode; + return exitCode; + } + else version (Windows) + { + int performWait(bool block) @trusted + { + if (_processID == terminated) return _exitCode; + assert (_handle != INVALID_HANDLE_VALUE); + if (block) + { + auto result = WaitForSingleObject(_handle, INFINITE); + if (result != WAIT_OBJECT_0) + throw ProcessException.newFromLastError("Wait failed."); + } + if (!GetExitCodeProcess(_handle, cast(LPDWORD)&_exitCode)) + throw ProcessException.newFromLastError(); + if (!block && _exitCode == STILL_ACTIVE) return 0; + CloseHandle(_handle); + _handle = INVALID_HANDLE_VALUE; + _processID = terminated; + return _exitCode; + } + + ~this() + { + if(_handle != INVALID_HANDLE_VALUE) + { + CloseHandle(_handle); + _handle = INVALID_HANDLE_VALUE; + } + } + } + + // Special values for _processID. + enum invalid = -1, terminated = -2; + + // OS process ID number. Only nonnegative IDs correspond to + // running processes. + int _processID = invalid; + + // Exit code cached by wait(). This is only expected to hold a + // sensible value if _processID == terminated. + int _exitCode; + + // Pids are only meant to be constructed inside this module, so + // we make the constructor private. + version (Windows) + { + HANDLE _handle; + this(int pid, HANDLE handle) @safe pure nothrow + { + _processID = pid; + _handle = handle; + } + } + else + { + this(int id) @safe pure nothrow + { + _processID = id; + } + } +} + + +/** +Waits for the process associated with $(D pid) to terminate, and returns +its exit status. + +In general one should always _wait for child processes to terminate +before exiting the parent process. Otherwise, they may become +"$(WEB en.wikipedia.org/wiki/Zombie_process,zombies)" – processes +that are defunct, yet still occupy a slot in the OS process table. + +If the process has already terminated, this function returns directly. +The exit code is cached, so that if wait() is called multiple times on +the same $(LREF Pid) it will always return the same value. + +POSIX_specific: +If the process is terminated by a signal, this function returns a +negative number whose absolute value is the signal number. +Since POSIX restricts normal exit codes to the range 0-255, a +negative return value will always indicate termination by signal. +Signal codes are defined in the $(D core.sys.posix.signal) module +(which corresponds to the $(D signal.h) POSIX header). + +Throws: +$(LREF ProcessException) on failure. + +Examples: +See the $(LREF spawnProcess) documentation. + +See_also: +$(LREF tryWait), for a non-blocking function. +*/ +int wait(Pid pid) @safe +{ + assert(pid !is null, "Called wait on a null Pid."); + return pid.performWait(true); +} + + +/** +A non-blocking version of $(LREF wait). + +If the process associated with $(D pid) has already terminated, +$(D tryWait) has the exact same effect as $(D wait). +In this case, it returns a tuple where the $(D terminated) field +is set to $(D true) and the $(D status) field has the same +interpretation as the return value of $(D wait). + +If the process has $(I not) yet terminated, this function differs +from $(D wait) in that does not wait for this to happen, but instead +returns immediately. The $(D terminated) field of the returned +tuple will then be set to $(D false), while the $(D status) field +will always be 0 (zero). $(D wait) or $(D tryWait) should then be +called again on the same $(D Pid) at some later time; not only to +get the exit code, but also to avoid the process becoming a "zombie" +when it finally terminates. (See $(LREF wait) for details). + +Throws: +$(LREF ProcessException) on failure. + +Example: +--- +auto pid = spawnProcess("dmd myapp.d"); +scope(exit) wait(pid); +... +auto dmd = tryWait(pid); +if (dmd.terminated) +{ + if (dmd.status == 0) writeln("Compilation succeeded!"); + else writeln("Compilation failed"); +} +else writeln("Still compiling..."); +... +--- +Note that in this example, the first $(D wait) call will have no +effect if the process has already terminated by the time $(D tryWait) +is called. In the opposite case, however, the $(D scope) statement +ensures that we always wait for the process if it hasn't terminated +by the time we reach the end of the scope. +*/ +Tuple!(bool, "terminated", int, "status") tryWait(Pid pid) @safe +{ + assert(pid !is null, "Called tryWait on a null Pid."); + auto code = pid.performWait(false); + return typeof(return)(pid._processID == Pid.terminated, code); +} + + +/** +Attempts to terminate the process associated with $(D pid). + +The effect of this function, as well as the meaning of $(D codeOrSignal), +is highly platform dependent. Details are given below. Common to all +platforms is that this function only $(I initiates) termination of the process, +and returns immediately. It does not wait for the process to end, +nor does it guarantee that the process does in fact get terminated. + +Always call $(LREF wait) to wait for a process to complete, even if $(D kill) +has been called on it. + +Windows_specific: +The process will be +$(LINK2 http://msdn.microsoft.com/en-us/library/windows/desktop/ms686714%28v=vs.100%29.aspx, +forcefully and abruptly terminated). If $(D codeOrSignal) is specified, it +will be used as the exit code of the process. If not, the process wil exit +with code 1. Do not use $(D codeOrSignal = 259), as this is a special value +(aka. $(LINK2 http://msdn.microsoft.com/en-us/library/windows/desktop/ms683189.aspx, +STILL_ACTIVE)) used by Windows to signal that a process has in fact $(I not) +terminated yet. +--- +auto pid = spawnProcess("some_app"); +kill(pid, 10); +assert (wait(pid) == 10); +--- + +POSIX_specific: +A $(LINK2 http://en.wikipedia.org/wiki/Unix_signal,signal) will be sent to +the process, whose value is given by $(D codeOrSignal). Depending on the +signal sent, this may or may not terminate the process. Symbolic constants +for various $(LINK2 http://en.wikipedia.org/wiki/Unix_signal#POSIX_signals, +POSIX signals) are defined in $(D core.sys.posix.signal), which corresponds to the +$(LINK2 http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html, +$(D signal.h) POSIX header). If $(D codeOrSignal) is omitted, the +$(D SIGTERM) signal will be sent. (This matches the behaviour of the +$(LINK2 http://pubs.opengroup.org/onlinepubs/9699919799/utilities/kill.html, +$(D _kill)) shell command.) +--- +import core.sys.posix.signal: SIGKILL; +auto pid = spawnProcess("some_app"); +kill(pid, SIGKILL); +assert (wait(pid) == -SIGKILL); // Negative return value on POSIX! +--- + +Throws: +$(LREF ProcessException) if the operating system reports an error. + (Note that this does not include failure to terminate the process, + which is considered a "normal" outcome.)$(BR) +$(OBJECTREF Error) if $(D codeOrSignal) is negative. +*/ +void kill(Pid pid) +{ + version (Windows) kill(pid, 1); + else version (Posix) + { + import core.sys.posix.signal: SIGTERM; + kill(pid, SIGTERM); + } +} + +/// ditto +void kill(Pid pid, int codeOrSignal) +{ + version (Windows) enum errMsg = "Invalid exit code"; + else version (Posix) enum errMsg = "Invalid signal"; + if (codeOrSignal < 0) throw new Error(errMsg); + + version (Windows) + { + if (!TerminateProcess(pid.osHandle, codeOrSignal)) + throw ProcessException.newFromLastError(); + } + else version (Posix) + { + import core.sys.posix.signal; + if (kill(pid.osHandle, codeOrSignal) == -1) + throw ProcessException.newFromErrno(); + } +} + +unittest +{ + // The test script goes into an infinite loop. + version (Windows) + { + TestScript prog = "loop: + goto loop"; + } + else version (Posix) + { + import core.sys.posix.signal: SIGTERM, SIGKILL; + TestScript prog = "while true; do; done"; + } + auto pid = spawnProcess(prog.path); + kill(pid); + version (Windows) assert (wait(pid) == 1); + else version (Posix) assert (wait(pid) == -SIGTERM); + + pid = spawnProcess(prog.path); + auto s = tryWait(pid); + assert (!s.terminated && s.status == 0); + version (Windows) kill(pid, 123); + else version (Posix) kill(pid, SIGKILL); + do { s = tryWait(pid); } while (!s.terminated); + version (Windows) assert (s.status == 123); + else version (Posix) assert (s.status == -SIGKILL); +} + + +private File encapPipeAsFile(FILE* fil) +{ + static struct Impl + { + FILE * handle = null; // Is null iff this Impl is closed by another File + uint refs = uint.max / 2; + bool isPipe; + } + auto f = File.wrapFile(fil); + auto imp = *cast(Impl**)&f; + imp.refs = 1; + imp.isPipe = true; + return f; +} + +/** +Creates a unidirectional _pipe. + +Data is written to one end of the _pipe and read from the other. +--- +auto p = pipe(); +p.writeEnd.writeln("Hello World"); +assert (p.readEnd.readln().chomp() == "Hello World"); +--- +Pipes can, for example, be used for interprocess communication +by spawning a new process and passing one end of the _pipe to +the child, while the parent uses the other end. +(See also $(LREF pipeProcess) and $(LREF pipeShell) for an easier +way of doing this.) +--- +// Use cURL to download the dlang.org front page, pipe its +// output to grep to extract a list of links to ZIP files, +// and write the list to the file "D downloads.txt": +auto p = pipe(); +auto outFile = File("D downloads.txt", "w"); +auto cpid = spawnProcess(["curl", "http://dlang.org/download.html"], + std.stdio.stdin, p.writeEnd); +scope(exit) wait(cpid); +auto gpid = spawnProcess(["grep", "-o", `http://\S*\.zip`], + p.readEnd, outFile); +scope(exit) wait(gpid); +--- + +Returns: +A $(LREF Pipe) object that corresponds to the created _pipe. + +Throws: +$(XREF stdio,StdioException) on failure. +*/ +version (Posix) +Pipe pipe() @trusted //TODO: @safe +{ + int[2] fds; + errnoEnforce(core.sys.posix.unistd.pipe(fds) == 0, + "Unable to create pipe"); + Pipe p; + auto readFP = fdopen(fds[0], "r"); + if (readFP == null) + throw new StdioException("Cannot open read end of pipe"); + p._read = encapPipeAsFile(readFP); + auto writeFP = fdopen(fds[1], "w"); + if (writeFP == null) + throw new StdioException("Cannot open write end of pipe"); + p._write = encapPipeAsFile(writeFP); + return p; +} +else version (Windows) +Pipe pipe() @trusted //TODO: @safe +{ + // use CreatePipe to create an anonymous pipe + HANDLE readHandle; + HANDLE writeHandle; + if (!CreatePipe(&readHandle, &writeHandle, null, 0)) + { + throw new StdioException( + "Error creating pipe (" ~ sysErrorString(GetLastError()) ~ ')', + 0); + } + + // Create file descriptors from the handles + version (DMC_RUNTIME) + { + auto readFD = _handleToFD(readHandle, FHND_DEVICE); + auto writeFD = _handleToFD(writeHandle, FHND_DEVICE); + } + else // MSVCRT + { + auto readFD = _open_osfhandle(readHandle, _O_RDONLY); + auto writeFD = _open_osfhandle(writeHandle, _O_APPEND); + } + version (DMC_RUNTIME) alias .close _close; + if (readFD == -1 || writeFD == -1) + { + // Close file descriptors, then throw. + if (readFD >= 0) _close(readFD); + else CloseHandle(readHandle); + if (writeFD >= 0) _close(writeFD); + else CloseHandle(writeHandle); + throw new StdioException("Error creating pipe"); + } + + // Create FILE pointers from the file descriptors + Pipe p; + version (DMC_RUNTIME) + { + // This is a re-implementation of DMC's fdopen, but without the + // mucking with the file descriptor. POSIX standard requires the + // new fdopen'd file to retain the given file descriptor's + // position. + FILE * local_fdopen(int fd, const(char)* mode) + { + auto fp = core.stdc.stdio.fopen("NUL", mode); + if(!fp) return null; + FLOCK(fp); + auto iob = cast(_iobuf*)fp; + .close(iob._file); + iob._file = fd; + iob._flag &= ~_IOTRAN; + FUNLOCK(fp); + return fp; + } + + auto readFP = local_fdopen(readFD, "r"); + auto writeFP = local_fdopen(writeFD, "a"); + } + else // MSVCRT + { + auto readFP = _fdopen(readFD, "r"); + auto writeFP = _fdopen(writeFD, "a"); + } + if (readFP == null || writeFP == null) + { + // Close streams, then throw. + if (readFP != null) fclose(readFP); + else _close(readFD); + if (writeFP != null) fclose(writeFP); + else _close(writeFD); + throw new StdioException("Cannot open pipe"); + } + p._read = encapPipeAsFile(readFP); + p._write = encapPipeAsFile(writeFP); + return p; +} + + +/// An interface to a pipe created by the $(LREF pipe) function. +struct Pipe +{ + /// The read end of the pipe. + @property File readEnd() @trusted /*TODO: @safe nothrow*/ { return _read; } + + + /// The write end of the pipe. + @property File writeEnd() @trusted /*TODO: @safe nothrow*/ { return _write; } + + + /** + Closes both ends of the pipe. + + Normally it is not necessary to do this manually, as $(XREF stdio,File) + objects are automatically closed when there are no more references + to them. + + Note that if either end of the pipe has been passed to a child process, + it will only be closed in the parent process. (What happens in the + child process is platform dependent.) + */ + void close() @trusted //TODO: @safe nothrow + { + _read.close(); + _write.close(); + } + +private: + File _read, _write; +} + +unittest +{ + auto p = pipe(); + p.writeEnd.writeln("Hello World"); + p.writeEnd.flush(); + assert (p.readEnd.readln().chomp() == "Hello World"); +} + + +/** +Starts a new process, creating pipes to redirect its standard +input, output and/or error streams. + +These functions return immediately, leaving the child process to +execute in parallel with the parent. +$(LREF pipeShell) invokes the user's _command interpreter, as +determined by $(LREF userShell), to execute the given program or +_command. + +Returns: +A $(LREF ProcessPipes) object which contains $(XREF stdio,File) +handles that communicate with the redirected streams of the child +process, along with the $(LREF Pid) of the process. + +Throws: +$(LREF ProcessException) on failure to start the process.$(BR) +$(XREF stdio,StdioException) on failure to create pipes.$(BR) +$(OBJECTREF Error) if $(D redirectFlags) is an invalid combination of flags. + +Example: +--- +auto pipes = pipeProcess("my_application", Redirect.stdout | Redirect.stderr); +scope(exit) wait(pipes.pid); + +// Store lines of output. +string[] output; +foreach (line; pipes.stdout.byLine) output ~= line.idup; + +// Store lines of errors. +string[] errors; +foreach (line; pipes.stderr.byLine) errors ~= line.idup; +--- +*/ +ProcessPipes pipeProcess(string program, + Redirect redirectFlags = Redirect.all) + @trusted +{ + return pipeProcessImpl!spawnProcess(program, redirectFlags); +} + +/// ditto +ProcessPipes pipeProcess(string[] args, + Redirect redirectFlags = Redirect.all) + @trusted //TODO: @safe +{ + return pipeProcessImpl!spawnProcess(args, redirectFlags); +} + +/// ditto +ProcessPipes pipeShell(string command, Redirect redirectFlags = Redirect.all) + @safe +{ + return pipeProcessImpl!spawnShell(command, redirectFlags); +} + +// Implementation of the pipeProcess() family of functions. +private ProcessPipes pipeProcessImpl(alias spawnFunc, Cmd) + (Cmd command, Redirect redirectFlags) + @trusted //TODO: @safe +{ + File childStdin, childStdout, childStderr; + ProcessPipes pipes; + pipes._redirectFlags = redirectFlags; + + if (redirectFlags & Redirect.stdin) + { + auto p = pipe(); + childStdin = p.readEnd; + pipes._stdin = p.writeEnd; + } + else + { + childStdin = std.stdio.stdin; + } + + if (redirectFlags & Redirect.stdout) + { + if ((redirectFlags & Redirect.stdoutToStderr) != 0) + throw new Error("Invalid combination of options: Redirect.stdout | " + ~"Redirect.stdoutToStderr"); + auto p = pipe(); + childStdout = p.writeEnd; + pipes._stdout = p.readEnd; + } + else + { + childStdout = std.stdio.stdout; + } + + if (redirectFlags & Redirect.stderr) + { + if ((redirectFlags & Redirect.stderrToStdout) != 0) + throw new Error("Invalid combination of options: Redirect.stderr | " + ~"Redirect.stderrToStdout"); + auto p = pipe(); + childStderr = p.writeEnd; + pipes._stderr = p.readEnd; + } + else + { + childStderr = std.stdio.stderr; + } + + if (redirectFlags & Redirect.stdoutToStderr) + { + if (redirectFlags & Redirect.stderrToStdout) + { + // We know that neither of the other options have been + // set, so we assign the std.stdio.std* streams directly. + childStdout = std.stdio.stderr; + childStderr = std.stdio.stdout; + } + else + { + childStdout = childStderr; + } + } + else if (redirectFlags & Redirect.stderrToStdout) + { + childStderr = childStdout; + } + + pipes._pid = spawnFunc(command, null, childStdin, childStdout, childStderr); + return pipes; +} + + +/** +Flags that can be passed to $(LREF pipeProcess) and $(LREF pipeShell) +to specify which of the child process' standard streams are redirected. +Use bitwise OR to combine flags. +*/ +enum Redirect +{ + /// Redirect the standard input, output or error streams, respectively. + stdin = 1, + stdout = 2, /// ditto + stderr = 4, /// ditto + + /** + Redirect _all three streams. This is equivalent to + $(D Redirect.stdin | Redirect.stdout | Redirect.stderr). + */ + all = stdin | stdout | stderr, + + /** + Redirect the standard error stream into the standard output stream. + This can not be combined with $(D Redirect.stderr). + */ + stderrToStdout = 8, + + /** + Redirect the standard output stream into the standard error stream. + This can not be combined with $(D Redirect.stdout). + */ + stdoutToStderr = 16, +} + +unittest +{ + version (Windows) TestScript prog = + "call :sub %1 %2 0 + call :sub %1 %2 1 + call :sub %1 %2 2 + call :sub %1 %2 3 + exit 3 + + :sub + set /p INPUT= + if -%INPUT%-==-stop- ( exit %3 ) + echo %INPUT% %1 + echo %INPUT% %2 1>&2"; + else version (Posix) TestScript prog = + `for EXITCODE in 0 1 2 3; do + read INPUT + if test "$INPUT" = stop; then break; fi + echo "$INPUT $1" + echo "$INPUT $2" >&2 + done + exit $EXITCODE`; + auto pp = pipeProcess([prog.path, "bar", "baz"]); + pp.stdin.writeln("foo"); + pp.stdin.flush(); + assert (pp.stdout.readln().chomp() == "foo bar"); + assert (pp.stderr.readln().chomp().stripRight() == "foo baz"); + pp.stdin.writeln("1234567890"); + pp.stdin.flush(); + assert (pp.stdout.readln().chomp() == "1234567890 bar"); + assert (pp.stderr.readln().chomp().stripRight() == "1234567890 baz"); + pp.stdin.writeln("stop"); + pp.stdin.flush(); + assert (wait(pp.pid) == 2); + + pp = pipeProcess([prog.path, "12345", "67890"], + Redirect.stdin | Redirect.stdout | Redirect.stderrToStdout); + pp.stdin.writeln("xyz"); + pp.stdin.flush(); + assert (pp.stdout.readln().chomp() == "xyz 12345"); + assert (pp.stdout.readln().chomp().stripRight() == "xyz 67890"); + pp.stdin.writeln("stop"); + pp.stdin.flush(); + assert (wait(pp.pid) == 1); + + pp = pipeShell(prog.path~" AAAAA BBB", + Redirect.stdin | Redirect.stdoutToStderr | Redirect.stderr); + pp.stdin.writeln("ab"); + pp.stdin.flush(); + assert (pp.stderr.readln().chomp() == "ab AAAAA"); + assert (pp.stderr.readln().chomp().stripRight() == "ab BBB"); + pp.stdin.writeln("stop"); + pp.stdin.flush(); + assert (wait(pp.pid) == 1); +} + + +/** +Object which contains $(XREF stdio,File) handles that allow communication +with a child process through its standard streams. +*/ +struct ProcessPipes +{ + /// The $(LREF Pid) of the child process. + @property Pid pid() @safe nothrow + { + assert(_pid !is null); + return _pid; + } + + /** + An $(XREF stdio,File) that allows writing to the child process' + standard input stream. + + Throws: + $(OBJECTREF Error) if the child process' standard input stream hasn't + been redirected. + */ + @property File stdin() @trusted //TODO: @safe nothrow + { + if ((_redirectFlags & Redirect.stdin) == 0) + throw new Error("Child process' standard input stream hasn't " + ~"been redirected."); + return _stdin; + } + + /** + An $(XREF stdio,File) that allows reading from the child process' + standard output stream. + + Throws: + $(OBJECTREF Error) if the child process' standard output stream hasn't + been redirected. + */ + @property File stdout() @trusted //TODO: @safe nothrow + { + if ((_redirectFlags & Redirect.stdout) == 0) + throw new Error("Child process' standard output stream hasn't " + ~"been redirected."); + return _stdout; + } + + /** + An $(XREF stdio,File) that allows reading from the child process' + standard error stream. + + Throws: + $(OBJECTREF Error) if the child process' standard error stream hasn't + been redirected. + */ + @property File stderr() @trusted //TODO: @safe nothrow + { + if ((_redirectFlags & Redirect.stderr) == 0) + throw new Error("Child process' standard error stream hasn't " + ~"been redirected."); + return _stderr; + } + +private: + Redirect _redirectFlags; + Pid _pid; + File _stdin, _stdout, _stderr; +} + + +/** +Executes the given program and returns its exit code and output. + +This function blocks until the program terminates. +The $(D output) string includes what the program writes to its +standard error stream as well as its standard output stream. +--- +auto dmd = execute("dmd", "myapp.d"); +if (dmd.status != 0) writeln("Compilation failed:\n", dmd.output); +--- + +POSIX_specific: +If the process is terminated by a signal, the $(D status) field of +the return value will contain a negative number whose absolute +value is the signal number. (See $(LREF wait) for details.) + +Throws: +$(LREF ProcessException) on failure to start the process.$(BR) +$(XREF stdio,StdioException) on failure to capture output. +*/ +Tuple!(int, "status", string, "output") execute(string[] args...) + @trusted //TODO: @safe +{ + auto p = pipeProcess(args, Redirect.stdout | Redirect.stderrToStdout); + return processOutput(p, size_t.max); +} + +unittest +{ + // To avoid printing the newline characters, we use the echo|set trick on + // Windows, and printf on POSIX (neither echo -n nor echo \c are portable). + version (Windows) TestScript prog = + "echo|set /p=%1 + echo|set /p=%2 1>&2 + exit 123"; + else version (Posix) TestScript prog = + `printf '%s' $1 + printf '%s' $2 >&2 + exit 123`; + auto r = execute([prog.path, "foo", "bar"]); + assert (r.status == 123); + assert (r.output.stripRight() == "foobar"); + auto s = execute(prog.path, "Hello", "World"); + assert (s.status == 123); + assert (s.output.stripRight() == "HelloWorld"); +} + + +/** +Executes $(D _command) in the user's default _shell and returns its +exit code and output. + +This function blocks until the command terminates. +The $(D output) string includes what the command writes to its +standard error stream as well as its standard output stream. +The path to the _command interpreter is given by $(LREF userShell). +--- +auto ls = shell("ls -l"); +writefln("ls exited with code %s and said: %s", ls.status, ls.output); +--- + +POSIX_specific: +If the process is terminated by a signal, the $(D status) field of +the return value will contain a negative number whose absolute +value is the signal number. (See $(LREF wait) for details.) + +Throws: +$(LREF ProcessException) on failure to start the process.$(BR) +$(XREF stdio,StdioException) on failure to capture output. +*/ +Tuple!(int, "status", string, "output") shell(string command) + @trusted //TODO: @safe +{ + auto p = pipeShell(command, Redirect.stdout | Redirect.stderrToStdout); + return processOutput(p, size_t.max); +} + +unittest +{ + auto r1 = shell("echo foo"); + assert (r1.status == 0); + assert (r1.output.chomp() == "foo"); + auto r2 = shell("echo bar 1>&2"); + assert (r2.status == 0); + assert (r2.output.chomp().stripRight() == "bar"); + auto r3 = shell("exit 123"); + assert (r3.status == 123); + assert (r3.output.empty); +} + +// Collects the output and exit code for execute() and shell(). +private Tuple!(int, "status", string, "output") processOutput( + ref ProcessPipes pp, + size_t maxData) +{ + Appender!(ubyte[]) a; + enum chunkSize = 4096; + foreach (ubyte[] chunk; pp.stdout.byChunk(chunkSize)) + { + a.put(chunk); + if (a.data().length + chunkSize > maxData) break; + } + + typeof(return) r; + r.output = cast(string) a.data; + r.status = wait(pp.pid); + return r; +} + + + +/// An exception that signals a problem with starting or waiting for a process. +class ProcessException : Exception +{ + // Standard constructor. + this(string msg, string file = __FILE__, size_t line = __LINE__) + { + super(msg, file, line); + } + + // Creates a new ProcessException based on errno. + static ProcessException newFromErrno(string customMsg = null, + string file = __FILE__, + size_t line = __LINE__) + { + import core.stdc.errno; + import std.c.string; + version (linux) + { + char[1024] buf; + auto errnoMsg = to!string( + std.c.string.strerror_r(errno, buf.ptr, buf.length)); + } + else + { + auto errnoMsg = to!string(std.c.string.strerror(errno)); + } + auto msg = customMsg.empty() ? errnoMsg + : customMsg ~ " (" ~ errnoMsg ~ ')'; + return new ProcessException(msg, file, line); + } + + // Creates a new ProcessException based on GetLastError() (Windows only). + version (Windows) + static ProcessException newFromLastError(string customMsg = null, + string file = __FILE__, + size_t line = __LINE__) + { + auto lastMsg = sysErrorString(GetLastError()); + auto msg = customMsg.empty() ? lastMsg + : customMsg ~ " (" ~ lastMsg ~ ')'; + return new ProcessException(msg, file, line); + } +} + + +/** +Determines the path to the current user's default command interpreter. + +On Windows, this function returns the contents of the COMSPEC environment +variable, if it exists. Otherwise, it returns the string $(D "cmd.exe"). + +On POSIX, $(D userShell) returns the contents of the SHELL environment +variable, if it exists and is non-empty. Otherwise, it returns +$(D "/bin/sh"). +*/ +@property string userShell() @safe //TODO: nothrow +{ + version (Windows) return environment.get("COMSPEC", "cmd.exe"); + else version (Posix) return environment.get("SHELL", "/bin/sh"); +} + + +// A command-line switch that indicates to the shell that it should +// interpret the following argument as a command to be executed. +version (Posix) private immutable string shellSwitch = "-c"; +version (Windows) private immutable string shellSwitch = "/C"; + + +/// Returns the process ID number of the current process. +@property int thisProcessID() @trusted //TODO: @safe nothrow +{ + version (Windows) return GetCurrentProcessId(); + else version (Posix) return getpid(); +} + + +// Unittest support code: TestScript takes a string that contains a +// shell script for the current platform, and writes it to a temporary +// file. On Windows the file name gets a .cmd extension, while on +// POSIX its executable permission bit is set. The file is +// automatically deleted when the object goes out of scope. +version (unittest) +private struct TestScript +{ + this(string code) + { + import std.ascii, std.file, std.uuid; + version (Windows) + { + auto ext = ".cmd"; + auto firstLine = "@echo off"; + } + else version (Posix) + { + auto ext = ""; + auto firstLine = "#!/bin/sh"; + } + path = buildPath(tempDir(), randomUUID().toString()~ext); + std.file.write(path, firstLine~std.ascii.newline~code~std.ascii.newline); + version (Posix) + { + import core.sys.posix.sys.stat; + chmod(toStringz(path), octal!777); + } + } + + ~this() + { + import std.file; + if (!path.empty && exists(path)) remove(path); + } + + string path; +} + + +// ============================================================================= +// Functions for shell command quoting/escaping. +// ============================================================================= + + +/* + Command line arguments exist in three forms: + 1) string or char* array, as received by main. + Also used internally on POSIX systems. + 2) Command line string, as used in Windows' + CreateProcess and CommandLineToArgvW functions. + A specific quoting and escaping algorithm is used + to distinguish individual arguments. + 3) Shell command string, as written at a shell prompt + or passed to cmd /C - this one may contain shell + control characters, e.g. > or | for redirection / + piping - thus, yet another layer of escaping is + used to distinguish them from program arguments. + + Except for escapeWindowsArgument, the intermediary + format (2) is hidden away from the user in this module. +*/ + +/** +Escapes an argv-style argument array to be used with $(LREF spawnShell), +$(LREF pipeShell) or $(LREF shell). +--- +string url = "http://dlang.org/"; +shell(escapeShellCommand("wget", url, "-O", "dlang-index.html")); +--- + +Concatenate multiple $(D escapeShellCommand) and +$(LREF escapeShellFileName) results to use shell redirection or +piping operators. +--- +shell( + escapeShellCommand("curl", "http://dlang.org/download.html") ~ + "|" ~ + escapeShellCommand("grep", "-o", `http://\S*\.zip`) ~ + ">" ~ + escapeShellFileName("D download links.txt")); +--- +*/ +string escapeShellCommand(in char[][] args...) + //TODO: @safe pure nothrow +{ + return escapeShellCommandString(escapeShellArguments(args)); +} + + +private string escapeShellCommandString(string command) + //TODO: @safe pure nothrow +{ + version (Windows) + return escapeWindowsShellCommand(command); + else + return command; +} + +string escapeWindowsShellCommand(in char[] command) + //TODO: @safe pure nothrow (prevented by Appender) +{ + auto result = appender!string(); + result.reserve(command.length); + + foreach (c; command) + switch (c) + { + case '\0': + assert(0, "Cannot put NUL in command line"); + case '\r': + case '\n': + assert(0, "CR/LF are not escapable"); + case '\x01': .. case '\x09': + case '\x0B': .. case '\x0C': + case '\x0E': .. case '\x1F': + case '"': + case '^': + case '&': + case '<': + case '>': + case '|': + result.put('^'); + goto default; + default: + result.put(c); + } + return result.data; +} + +private string escapeShellArguments(in char[][] args...) + @trusted pure nothrow +{ + char[] buf; + + @safe nothrow + char[] allocator(size_t size) + { + if (buf.length == 0) + return buf = new char[size]; + else + { + auto p = buf.length; + buf.length = buf.length + 1 + size; + buf[p++] = ' '; + return buf[p..p+size]; + } + } + + foreach (arg; args) + escapeShellArgument!allocator(arg); + return assumeUnique(buf); +} + +private auto escapeShellArgument(alias allocator)(in char[] arg) @safe nothrow +{ + // The unittest for this function requires special + // preparation - see below. + + version (Windows) + return escapeWindowsArgumentImpl!allocator(arg); + else + return escapePosixArgumentImpl!allocator(arg); +} + +/** +Quotes a command-line argument in a manner conforming to the behavior of +$(LINK2 http://msdn.microsoft.com/en-us/library/windows/desktop/bb776391(v=vs.85).aspx, +CommandLineToArgvW). +*/ +string escapeWindowsArgument(in char[] arg) @trusted pure nothrow +{ + // Rationale for leaving this function as public: + // this algorithm of escaping paths is also used in other software, + // e.g. DMD's response files. + + auto buf = escapeWindowsArgumentImpl!charAllocator(arg); + return assumeUnique(buf); +} + + +private char[] charAllocator(size_t size) @safe pure nothrow +{ + return new char[size]; +} + + +private char[] escapeWindowsArgumentImpl(alias allocator)(in char[] arg) + @safe nothrow + if (is(typeof(allocator(size_t.init)[0] = char.init))) +{ + // References: + // * http://msdn.microsoft.com/en-us/library/windows/desktop/bb776391(v=vs.85).aspx + // * http://blogs.msdn.com/b/oldnewthing/archive/2010/09/17/10063629.aspx + + // Calculate the total string size. + + // Trailing backslashes must be escaped + bool escaping = true; + // Result size = input size + 2 for surrounding quotes + 1 for the + // backslash for each escaped character. + size_t size = 1 + arg.length + 1; + + foreach_reverse (c; arg) + { + if (c == '"') + { + escaping = true; + size++; + } + else + if (c == '\\') + { + if (escaping) + size++; + } + else + escaping = false; + } + + // Construct result string. + + auto buf = allocator(size); + size_t p = size; + buf[--p] = '"'; + escaping = true; + foreach_reverse (c; arg) + { + if (c == '"') + escaping = true; + else + if (c != '\\') + escaping = false; + + buf[--p] = c; + if (escaping) + buf[--p] = '\\'; + } + buf[--p] = '"'; + assert(p == 0); + + return buf; +} + +version(Windows) version(unittest) +{ + import core.sys.windows.windows; + import core.stdc.stddef; + + extern (Windows) wchar_t** CommandLineToArgvW(wchar_t*, int*); + extern (C) size_t wcslen(in wchar *); + + unittest + { + string[] testStrings = [ + `Hello`, + `Hello, world`, + `Hello, "world"`, + `C:\`, + `C:\dmd`, + `C:\Program Files\`, + ]; + + enum CHARS = `_x\" *&^`; // _ is placeholder for nothing + foreach (c1; CHARS) + foreach (c2; CHARS) + foreach (c3; CHARS) + foreach (c4; CHARS) + testStrings ~= [c1, c2, c3, c4].replace("_", ""); + + foreach (s; testStrings) + { + auto q = escapeWindowsArgument(s); + LPWSTR lpCommandLine = (to!(wchar[])("Dummy.exe " ~ q) ~ "\0"w).ptr; + int numArgs; + LPWSTR* args = CommandLineToArgvW(lpCommandLine, &numArgs); + scope(exit) LocalFree(args); + assert(numArgs==2, s ~ " => " ~ q ~ " #" ~ text(numArgs-1)); + auto arg = to!string(args[1][0..wcslen(args[1])]); + assert(arg == s, s ~ " => " ~ q ~ " => " ~ arg); + } + } +} + +private string escapePosixArgument(in char[] arg) @trusted pure nothrow +{ + auto buf = escapePosixArgumentImpl!charAllocator(arg); + return assumeUnique(buf); +} + +private char[] escapePosixArgumentImpl(alias allocator)(in char[] arg) + @safe nothrow + if (is(typeof(allocator(size_t.init)[0] = char.init))) +{ + // '\'' means: close quoted part of argument, append an escaped + // single quote, and reopen quotes + + // Below code is equivalent to: + // return `'` ~ std.array.replace(arg, `'`, `'\''`) ~ `'`; + + size_t size = 1 + arg.length + 1; + foreach (c; arg) + if (c == '\'') + size += 3; + + auto buf = allocator(size); + size_t p = 0; + buf[p++] = '\''; + foreach (c; arg) + if (c == '\'') + { + buf[p..p+4] = `'\''`; + p += 4; + } + else + buf[p++] = c; + buf[p++] = '\''; + assert(p == size); + + return buf; +} + +/** +Escapes a filename to be used for shell redirection with $(LREF spawnShell), +$(LREF pipeShell) or $(LREF shell). +*/ +string escapeShellFileName(in char[] fileName) @trusted pure nothrow +{ + // The unittest for this function requires special + // preparation - see below. + + version (Windows) + return cast(string)('"' ~ fileName ~ '"'); + else + return escapePosixArgument(fileName); +} + +// Loop generating strings with random characters +//version = unittest_burnin; + +version(unittest_burnin) +unittest +{ + // There are no readily-available commands on all platforms suitable + // for properly testing command escaping. The behavior of CMD's "echo" + // built-in differs from the POSIX program, and Windows ports of POSIX + // environments (Cygwin, msys, gnuwin32) may interfere with their own + // "echo" ports. + + // To run this unit test, create std_process_unittest_helper.d with the + // following content and compile it: + // import std.stdio, std.array; void main(string[] args) { write(args.join("\0")); } + // Then, test this module with: + // rdmd --main -unittest -version=unittest_burnin process.d + + auto helper = absolutePath("std_process_unittest_helper"); + assert(shell(helper ~ " hello").split("\0")[1..$] == ["hello"], "Helper malfunction"); + + void test(string[] s, string fn) + { + string e; + string[] g; + + e = escapeShellCommand(helper ~ s); + { + scope(failure) writefln("shell() failed.\nExpected:\t%s\nEncoded:\t%s", s, [e]); + g = shell(e).split("\0")[1..$]; + } + assert(s == g, format("shell() test failed.\nExpected:\t%s\nGot:\t\t%s\nEncoded:\t%s", s, g, [e])); + + e = escapeShellCommand(helper ~ s) ~ ">" ~ escapeShellFileName(fn); + { + scope(failure) writefln("system() failed.\nExpected:\t%s\nFilename:\t%s\nEncoded:\t%s", s, [fn], [e]); + system(e); + g = readText(fn).split("\0")[1..$]; + } + remove(fn); + assert(s == g, format("system() test failed.\nExpected:\t%s\nGot:\t\t%s\nEncoded:\t%s", s, g, [e])); + } + + while (true) + { + string[] args; + foreach (n; 0..uniform(1, 4)) + { + string arg; + foreach (l; 0..uniform(0, 10)) + { + dchar c; + while (true) + { + version (Windows) + { + // As long as DMD's system() uses CreateProcessA, + // we can't reliably pass Unicode + c = uniform(0, 128); + } + else + c = uniform!ubyte(); + + if (c == 0) + continue; // argv-strings are zero-terminated + version (Windows) + if (c == '\r' || c == '\n') + continue; // newlines are unescapable on Windows + break; + } + arg ~= c; + } + args ~= arg; + } + + // generate filename + string fn = "test_"; + foreach (l; 0..uniform(1, 10)) + { + dchar c; + while (true) + { + version (Windows) + c = uniform(0, 128); // as above + else + c = uniform!ubyte(); + + if (c == 0 || c == '/') + continue; // NUL and / are the only characters + // forbidden in POSIX filenames + version (Windows) + if (c < '\x20' || c == '<' || c == '>' || c == ':' || + c == '"' || c == '\\' || c == '|' || c == '?' || c == '*') + continue; // http://msdn.microsoft.com/en-us/library/aa365247(VS.85).aspx + break; + } + + fn ~= c; + } + + test(args, fn); + } +} + + +// ============================================================================= +// Environment variable manipulation. +// ============================================================================= + + +/** +Manipulates _environment variables using an associative-array-like +interface. + +This class contains only static methods, and cannot be instantiated. +See below for examples of use. +*/ +abstract final class environment +{ +static: + /** + Retrieves the value of the environment variable with the given $(D name). + + If no such variable exists, this function throws an $(D Exception). + See also $(LREF get), which doesn't throw on failure. + --- + auto path = environment["PATH"]; + --- + */ + string opIndex(string name) @safe + { + string value; + enforce(getImpl(name, value), "Environment variable not found: "~name); + return value; + } + + /** + Retrieves the value of the environment variable with the given $(D name), + or a default value if the variable doesn't exist. + + Unlike $(LREF opIndex), this function never throws. + --- + auto sh = environment.get("SHELL", "/bin/sh"); + --- + This function is also useful in checking for the existence of an + environment variable. + --- + auto myVar = environment.get("MYVAR"); + if (myVar is null) + { + // Environment variable doesn't exist. + // Note that we have to use 'is' for the comparison, since + // myVar == null is also true if the variable exists but is + // empty. + } + --- + */ + string get(string name, string defaultValue = null) @safe //TODO: nothrow + { + string value; + auto found = getImpl(name, value); + return found ? value : defaultValue; + } + + /** + Assigns the given $(D value) to the environment variable with the given + $(D name). + + If the variable does not exist, it will be created. If it already exists, + it will be overwritten. + --- + environment["foo"] = "bar"; + --- + */ + string opIndexAssign(string value, string name) @trusted + { + version (Posix) + { + if (core.sys.posix.stdlib.setenv(toStringz(name), toStringz(value), 1) != -1) + { + return value; + } + // The default errno error message is very uninformative + // in the most common case, so we handle it manually. + enforce(errno != EINVAL, + "Invalid environment variable name: '"~name~"'"); + errnoEnforce(false, + "Failed to add environment variable"); + assert(0); + } + else version (Windows) + { + enforce( + SetEnvironmentVariableW(toUTF16z(name), toUTF16z(value)), + sysErrorString(GetLastError()) + ); + return value; + } + else static assert(0); + } + + /** + Removes the environment variable with the given $(D name). + + If the variable isn't in the environment, this function returns + successfully without doing anything. + */ + void remove(string name) @trusted // TODO: @safe nothrow + { + version (Windows) SetEnvironmentVariableW(toUTF16z(name), null); + else version (Posix) core.sys.posix.stdlib.unsetenv(toStringz(name)); + else static assert(0); + } + + /** + Copies all environment variables into an associative array. + + Windows_specific: + While Windows environment variable names are case insensitive, D's + built-in associative arrays are not. This function will store all + variable names in uppercase (e.g. $(D PATH)). + */ + string[string] toAA() @trusted + { + string[string] aa; + version (Posix) + { + for (int i=0; environ[i] != null; ++i) + { + immutable varDef = to!string(environ[i]); + immutable eq = std.string.indexOf(varDef, '='); + assert (eq >= 0); + + immutable name = varDef[0 .. eq]; + immutable value = varDef[eq+1 .. $]; + + // In POSIX, environment variables may be defined more + // than once. This is a security issue, which we avoid + // by checking whether the key already exists in the array. + // For more info: + // http://www.dwheeler.com/secure-programs/Secure-Programs-HOWTO/environment-variables.html + if (name !in aa) aa[name] = value; + } + } + else version (Windows) + { + auto envBlock = GetEnvironmentStringsW(); + enforce(envBlock, "Failed to retrieve environment variables."); + scope(exit) FreeEnvironmentStringsW(envBlock); + + for (int i=0; envBlock[i] != '\0'; ++i) + { + auto start = i; + while (envBlock[i] != '=') ++i; + immutable name = toUTF8(toUpper(envBlock[start .. i])); + + start = i+1; + while (envBlock[i] != '\0') ++i; + // Just like in POSIX systems, environment variables may be + // defined more than once in an environment block on Windows, + // and it is just as much of a security issue there. Moreso, + // in fact, due to the case insensensitivity of variable names, + // which is not handled correctly by all programs. + if (name !in aa) aa[name] = toUTF8(envBlock[start .. i]); + } + } + else static assert(0); + return aa; + } + +private: + // Returns the length of an environment variable (in number of + // wchars, including the null terminator), or 0 if it doesn't exist. + version (Windows) + int varLength(LPCWSTR namez) @trusted nothrow + { + return GetEnvironmentVariableW(namez, null, 0); + } + + // Retrieves the environment variable, returns false on failure. + bool getImpl(string name, out string value) @trusted //TODO: nothrow + { + version (Windows) + { + const namez = toUTF16z(name); + immutable len = varLength(namez); + if (len == 0) return false; + if (len == 1) + { + value = ""; + return true; + } + + auto buf = new WCHAR[len]; + GetEnvironmentVariableW(namez, buf.ptr, to!DWORD(buf.length)); + value = toUTF8(buf[0 .. $-1]); + return true; + } + else version (Posix) + { + const vz = core.sys.posix.stdlib.getenv(toStringz(name)); + if (vz == null) return false; + auto v = vz[0 .. strlen(vz)]; + + // Cache the last call's result. + static string lastResult; + if (v != lastResult) lastResult = v.idup; + value = lastResult; + return true; + } + else static assert(0); + } +} + +unittest +{ + // New variable + environment["std_process"] = "foo"; + assert (environment["std_process"] == "foo"); + + // Set variable again + environment["std_process"] = "bar"; + assert (environment["std_process"] == "bar"); + + // Remove variable + environment.remove("std_process"); + + // Remove again, should succeed + environment.remove("std_process"); + + // Throw on not found. + assertThrown(environment["std_process"]); + + // get() without default value + assert (environment.get("std_process") == null); + + // get() with default value + assert (environment.get("std_process", "baz") == "baz"); + + // Convert to associative array + auto aa = environment.toAA(); + assert (aa.length > 0); + foreach (n, v; aa) + { + // Wine has some bugs related to environment variables: + // - Wine allows the existence of an env. variable with the name + // "\0", but GetEnvironmentVariable refuses to retrieve it. + // - If an env. variable has zero length, i.e. is "\0", + // GetEnvironmentVariable should return 1. Instead it returns + // 0, indicating the variable doesn't exist. + version (Windows) if (n.length == 0 || v.length == 0) continue; + + assert (v == environment[n]); + } +} diff --git a/source/dub/internal/vibecompat/core/file.d b/source/dub/internal/vibecompat/core/file.d new file mode 100644 index 0000000..0444b14 --- /dev/null +++ b/source/dub/internal/vibecompat/core/file.d @@ -0,0 +1,291 @@ +/** + File handling. + + Copyright: © 2012 RejectedSoftware e.K. + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Sönke Ludwig +*/ +module dub.internal.vibecompat.core.file; + +public import dub.internal.vibecompat.inet.url; +public import std.stdio; + +import dub.internal.vibecompat.core.log; + +import std.conv; +import std.c.stdio; +import std.datetime; +import std.exception; +import std.file; +import std.path; +import std.string; +import std.utf; + + +version(Posix){ + private extern(C) int mkstemps(char* templ, int suffixlen); +} + + +/* Add output range support to File +*/ +struct RangeFile { + File file; + alias file this; + + void put(in ubyte[] bytes) { file.rawWrite(bytes); } + void put(in char[] str) { put(cast(ubyte[])str); } + void put(char ch) { put((&ch)[0 .. 1]); } + void put(dchar ch) { char[4] chars; put(chars[0 .. encode(chars, ch)]); } + + ubyte[] readAll() + { + file.seek(0, SEEK_END); + auto sz = file.tell(); + enforce(sz <= size_t.max, "File is too big to read to memory."); + file.seek(0, SEEK_SET); + auto ret = new ubyte[cast(size_t)sz]; + return file.rawRead(ret); + } +} + + +/** + Opens a file stream with the specified mode. +*/ +RangeFile openFile(Path path, FileMode mode = FileMode.Read) +{ + string strmode; + final switch(mode){ + case FileMode.Read: strmode = "rb"; break; + case FileMode.ReadWrite: strmode = "rb+"; break; + case FileMode.CreateTrunc: strmode = "wb+"; break; + case FileMode.Append: strmode = "ab"; break; + } + auto ret = File(path.toNativeString(), strmode); + assert(ret.isOpen()); + return RangeFile(ret); +} +/// ditto +RangeFile openFile(string path, FileMode mode = FileMode.Read) +{ + return openFile(Path(path), mode); +} + +/** + Creates and opens a temporary file for writing. +*/ +RangeFile createTempFile(string suffix = null) +{ + version(Windows){ + char[L_tmpnam] tmp; + tmpnam(tmp.ptr); + auto tmpname = to!string(tmp.ptr); + if( tmpname.startsWith("\\") ) tmpname = tmpname[1 .. $]; + tmpname ~= suffix; + logDebug("tmp %s", tmpname); + return openFile(tmpname, FileMode.CreateTrunc); + } else { + import core.sys.posix.stdio; + enum pattern ="/tmp/vtmp.XXXXXX"; + scope templ = new char[pattern.length+suffix.length+1]; + templ[0 .. pattern.length] = pattern; + templ[pattern.length .. $-1] = suffix; + templ[$-1] = '\0'; + assert(suffix.length <= int.max); + auto fd = mkstemps(templ.ptr, cast(int)suffix.length); + enforce(fd >= 0, "Failed to create temporary file."); + auto ret = File.wrapFile(fdopen(fd, "wb+")); + return RangeFile(ret); + } +} + +/** + Moves or renames a file. +*/ +void moveFile(Path from, Path to) +{ + moveFile(from.toNativeString(), to.toNativeString()); +} +/// ditto +void moveFile(string from, string to) +{ + std.file.rename(from, to); +} + +/** + Copies a file. + + Note that attributes and time stamps are currently not retained. + + Params: + from = Path of the source file + to = Path for the destination file + overwrite = If true, any file existing at the destination path will be + overwritten. If this is false, an excpetion will be thrown should + a file already exist at the destination path. + + Throws: + An Exception if the copy operation fails for some reason. +*/ +void copyFile(Path from, Path to, bool overwrite = false) +{ + enforce(overwrite || !existsFile(to), "Destination file already exists."); + .copy(from.toNativeString(), to.toNativeString()); +} +/// ditto +void copyFile(string from, string to) +{ + copyFile(Path(from), Path(to)); +} + +/** + Removes a file +*/ +void removeFile(Path path) +{ + removeFile(path.toNativeString()); +} +/// ditto +void removeFile(string path) { + std.file.remove(path); +} + +/** + Checks if a file exists +*/ +bool existsFile(Path path) { + return existsFile(path.toNativeString()); +} +/// ditto +bool existsFile(string path) +{ + return std.file.exists(path); +} + +/** Stores information about the specified file/directory into 'info' + + Returns false if the file does not exist. +*/ +FileInfo getFileInfo(Path path) +{ + auto ent = std.file.dirEntry(path.toNativeString()); + return makeFileInfo(ent); +} +/// ditto +FileInfo getFileInfo(string path) +{ + return getFileInfo(Path(path)); +} + +/** + Creates a new directory. +*/ +void createDirectory(Path path) +{ + mkdir(path.toNativeString()); +} +/// ditto +void createDirectory(string path) +{ + createDirectory(Path(path)); +} + +/** + Enumerates all files in the specified directory. +*/ +void listDirectory(Path path, scope bool delegate(FileInfo info) del) +{ + foreach( DirEntry ent; dirEntries(path.toNativeString(), SpanMode.shallow) ) + if( !del(makeFileInfo(ent)) ) + break; +} +/// ditto +void listDirectory(string path, scope bool delegate(FileInfo info) del) +{ + listDirectory(Path(path), del); +} +/// ditto +int delegate(scope int delegate(ref FileInfo)) iterateDirectory(Path path) +{ + int iterator(scope int delegate(ref FileInfo) del){ + int ret = 0; + listDirectory(path, (fi){ + ret = del(fi); + return ret == 0; + }); + return ret; + } + return &iterator; +} +/// ditto +int delegate(scope int delegate(ref FileInfo)) iterateDirectory(string path) +{ + return iterateDirectory(Path(path)); +} + + +/** + Returns the current working directory. +*/ +Path getWorkingDirectory() +{ + return Path(std.file.getcwd()); +} + + +/** Contains general information about a file. +*/ +struct FileInfo { + /// Name of the file (not including the path) + string name; + + /// Size of the file (zero for directories) + ulong size; + + /// Time of the last modification + SysTime timeModified; + + /// Time of creation (not available on all operating systems/file systems) + SysTime timeCreated; + + /// True if this is a symlink to an actual file + bool isSymlink; + + /// True if this is a directory or a symlink pointing to a directory + bool isDirectory; +} + +/** + Specifies how a file is manipulated on disk. +*/ +enum FileMode { + /// The file is opened read-only. + Read, + /// The file is opened for read-write random access. + ReadWrite, + /// The file is truncated if it exists and created otherwise and the opened for read-write access. + CreateTrunc, + /// The file is opened for appending data to it and created if it does not exist. + Append +} + +/** + Accesses the contents of a file as a stream. +*/ + +private FileInfo makeFileInfo(DirEntry ent) +{ + FileInfo ret; + ret.name = baseName(ent.name); + if( ret.name.length == 0 ) ret.name = ent.name; + assert(ret.name.length > 0); + ret.size = ent.size; + ret.timeModified = ent.timeLastModified; + version(Windows) ret.timeCreated = ent.timeCreated; + else ret.timeCreated = ent.timeLastModified; + ret.isSymlink = ent.isSymlink; + ret.isDirectory = ent.isDir; + return ret; +} + diff --git a/source/dub/internal/vibecompat/core/log.d b/source/dub/internal/vibecompat/core/log.d new file mode 100644 index 0000000..fc6ad80 --- /dev/null +++ b/source/dub/internal/vibecompat/core/log.d @@ -0,0 +1,97 @@ +/** + Central logging facility for vibe. + + Copyright: © 2012 RejectedSoftware e.K. + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Sönke Ludwig +*/ +module dub.internal.vibecompat.core.log; + +import std.array; +import std.datetime; +import std.format; +import std.stdio; +import core.thread; + +private { + shared LogLevel s_minLevel = LogLevel.Info; + shared LogLevel s_logFileLevel; + shared bool s_plainLogging = false; +} + +/// Sets the minimum log level to be printed. +void setLogLevel(LogLevel level) nothrow +{ + s_minLevel = level; +} + +/// Disables output of thread/task ids with each log message +void setPlainLogging(bool enable) +{ + s_plainLogging = enable; +} + +/** + Logs a message. + + Params: + level = The log level for the logged message + fmt = See http://dlang.org/phobos/std_format.html#format-string +*/ +void logTrace(T...)(string fmt, auto ref T args) nothrow { log(LogLevel.Trace, fmt, args); } +/// ditto +void logDebug(T...)(string fmt, auto ref T args) nothrow { log(LogLevel.Debug, fmt, args); } +/// ditto +void logInfo(T...)(string fmt, auto ref T args) nothrow { log(LogLevel.Info, fmt, args); } +/// ditto +void logWarn(T...)(string fmt, auto ref T args) nothrow { log(LogLevel.Warn, fmt, args); } +/// ditto +void logError(T...)(string fmt, auto ref T args) nothrow { log(LogLevel.Error, fmt, args); } + +/// ditto +void log(T...)(LogLevel level, string fmt, auto ref T args) +nothrow { + if( level < s_minLevel ) return; + string pref; + final switch( level ){ + case LogLevel.Trace: pref = "trc"; break; + case LogLevel.Debug: pref = "dbg"; break; + case LogLevel.Info: pref = "INF"; break; + case LogLevel.Warn: pref = "WRN"; break; + case LogLevel.Error: pref = "ERR"; break; + case LogLevel.Fatal: pref = "FATAL"; break; + case LogLevel.None: assert(false); + } + + try { + auto txt = appender!string(); + txt.reserve(256); + formattedWrite(txt, fmt, args); + + auto threadid = cast(ulong)cast(void*)Thread.getThis(); + auto fiberid = cast(ulong)cast(void*)Fiber.getThis(); + threadid ^= threadid >> 32; + fiberid ^= fiberid >> 32; + + if( level >= s_minLevel ){ + if( s_plainLogging ) writeln(txt.data()); + else writefln("[%08X:%08X %s] %s", threadid, fiberid, pref, txt.data()); + stdout.flush(); + } + } catch( Exception e ){ + // this is bad but what can we do.. + debug assert(false, e.msg); + } +} + +/// Specifies the log level for a particular log message. +enum LogLevel { + Trace, + Debug, + Info, + Warn, + Error, + Fatal, + None +} + diff --git a/source/dub/internal/vibecompat/data/json.d b/source/dub/internal/vibecompat/data/json.d new file mode 100644 index 0000000..6fd692b --- /dev/null +++ b/source/dub/internal/vibecompat/data/json.d @@ -0,0 +1,1287 @@ +/** + JSON serialization and value handling. + + This module provides the Json struct for reading, writing and manipulating JSON values in a seamless, + JavaScript like way. De(serialization) of arbitrary D types is also supported. + + Examples: + + --- + void manipulateJson(Json j) + { + // object members can be accessed using member syntax, just like in JavaScript + j = Json.EmptyObject; + j.name = "Example"; + j.id = 1; + + // retrieving the values is done using get() + assert(j["name"].get!string == "Example"); + assert(j["id"].get!int == 1); + + // semantic convertions can be done using to() + assert(j.id.to!string == "1"); + + // prints: + // name: "Example" + // id: 1 + foreach( string key, value; j ){ + writefln("%s: %s", key, value); + } + + // print out as JSON: {"name": "Example", "id": 1} + writefln("JSON: %s", j.toString()); + } + --- + + Copyright: © 2012 RejectedSoftware e.K. + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Sönke Ludwig +*/ +module dub.internal.vibecompat.data.json; + +import dub.internal.vibecompat.data.utils; + +import std.array; +import std.conv; +import std.datetime; +import std.exception; +import std.format; +import std.string; +import std.range; +import std.traits; + + +/******************************************************************************/ +/* public types */ +/******************************************************************************/ + +/** + Represents a single JSON value. + + Json values can have one of the types defined in the Json.Type enum. They + behave mostly like values in ECMA script in the way that you can + transparently perform operations on them. However, strict typechecking is + done, so that operations between differently typed JSON values will throw + an exception. Additionally, an explicit cast or using get!() or to!() is + required to convert a JSON value to the corresponding static D type. +*/ +struct Json { + private { + union { + bool m_bool; + long m_int; + double m_float; + string m_string; + Json[] m_array; + Json[string] m_object; + }; + Type m_type = Type.Undefined; + } + + /** Represents the run time type of a JSON value. + */ + enum Type { + /// A non-existent value in a JSON object + Undefined, + /// Null value + Null, + /// Boolean value + Bool, + /// 64-bit integer value + Int, + /// 64-bit floating point value + Float, + /// UTF-8 string + String, + /// Array of JSON values + Array, + /// JSON object aka. dictionary from string to Json + Object + } + + /// New JSON value of Type.Undefined + static @property Json Undefined() { return Json(); } + + /// New JSON value of Type.Object + static @property Json EmptyObject() { return Json(cast(Json[string])null); } + + /// New JSON value of Type.Array + static @property Json EmptyArray() { return Json(cast(Json[])null); } + + version(JsonLineNumbers) int line; + + /** + Constructor for a JSON object. + */ + this(typeof(null)) { m_type = Type.Null; } + /// ditto + this(bool v) { m_type = Type.Bool; m_bool = v; } + /// ditto + this(int v) { m_type = Type.Int; m_int = v; } + /// ditto + this(long v) { m_type = Type.Int; m_int = v; } + /// ditto + this(double v) { m_type = Type.Float; m_float = v; } + /// ditto + this(string v) { m_type = Type.String; m_string = v; } + /// ditto + this(Json[] v) { m_type = Type.Array; m_array = v; } + /// ditto + this(Json[string] v) { m_type = Type.Object; m_object = v; } + + /** + Allows assignment of D values to a JSON value. + */ + ref Json opAssign(Json v){ + m_type = v.m_type; + final switch(m_type){ + case Type.Undefined: m_string = null; break; + case Type.Null: m_string = null; break; + case Type.Bool: m_bool = v.m_bool; break; + case Type.Int: m_int = v.m_int; break; + case Type.Float: m_float = v.m_float; break; + case Type.String: m_string = v.m_string; break; + case Type.Array: m_array = v.m_array; break; + case Type.Object: m_object = v.m_object; break; + } + return this; + } + /// ditto + void opAssign(typeof(null)) { m_type = Type.Null; m_string = null; } + /// ditto + bool opAssign(bool v) { m_type = Type.Bool; m_bool = v; return v; } + /// ditto + int opAssign(int v) { m_type = Type.Int; m_int = v; return v; } + /// ditto + long opAssign(long v) { m_type = Type.Int; m_int = v; return v; } + /// ditto + double opAssign(double v) { m_type = Type.Float; m_float = v; return v; } + /// ditto + string opAssign(string v) { m_type = Type.String; m_string = v; return v; } + /// ditto + Json[] opAssign(Json[] v) { m_type = Type.Array; m_array = v; return v; } + /// ditto + Json[string] opAssign(Json[string] v) { m_type = Type.Object; m_object = v; return v; } + + /** + The current type id of this JSON object. + */ + @property Type type() const { return m_type; } + + /** + Allows direct indexing of array typed JSON values. + */ + ref inout(Json) opIndex(size_t idx) inout { checkType!(Json[])(); return m_array[idx]; } + + /** + Allows direct indexing of object typed JSON values using a string as + the key. + */ + const(Json) opIndex(string key) const { + checkType!(Json[string])(); + if( auto pv = key in m_object ) return *pv; + Json ret = Json.Undefined; + ret.m_string = key; + return ret; + } + /// ditto + ref Json opIndex(string key){ + checkType!(Json[string])(); + if( auto pv = key in m_object ) + return *pv; + m_object[key] = Json(); + m_object[key].m_type = Type.Undefined; // DMDBUG: AAs are teh $H1T!!!11 + assert(m_object[key].type == Type.Undefined); + m_object[key].m_string = key; + return m_object[key]; + } + + /** + Returns a slice of a JSON array. + */ + inout(Json[]) opSlice() inout { checkType!(Json[])(); return m_array; } + /// + inout(Json[]) opSlice(size_t from, size_t to) inout { checkType!(Json[])(); return m_array[from .. to]; } + + /** + Removes an entry from an object. + */ + void remove(string item) { checkType!(Json[string])(); m_object.remove(item); } + + /** + Returns the number of entries of string, array or object typed JSON values. + */ + @property size_t length() + const { + switch(m_type){ + case Type.String: return m_string.length; + case Type.Array: return m_array.length; + case Type.Object: return m_object.length; + default: + enforce(false, "Json.length() can only be called on strings, arrays and objects, not "~.to!string(m_type)~"."); + return 0; + } + } + + /** + Allows foreach iterating over JSON objects and arrays. + */ + int opApply(int delegate(ref Json obj) del) + { + enforce(m_type == Type.Array || m_type == Type.Object, "opApply may only be called on objects and arrays, not "~.to!string(m_type)~"."); + if( m_type == Type.Array ){ + foreach( ref v; m_array ) + if( auto ret = del(v) ) + return ret; + return 0; + } else { + foreach( ref v; m_object ) + if( v.type != Type.Undefined ) + if( auto ret = del(v) ) + return ret; + return 0; + } + } + /// ditto + int opApply(int delegate(ref const Json obj) del) + const { + enforce(m_type == Type.Array || m_type == Type.Object, "opApply may only be called on objects and arrays, not "~.to!string(m_type)~"."); + if( m_type == Type.Array ){ + foreach( ref v; m_array ) + if( auto ret = del(v) ) + return ret; + return 0; + } else { + foreach( ref v; m_object ) + if( v.type != Type.Undefined ) + if( auto ret = del(v) ) + return ret; + return 0; + } + } + /// ditto + int opApply(int delegate(ref size_t idx, ref Json obj) del) + { + enforce(m_type == Type.Array, "opApply may only be called on arrays, not "~.to!string(m_type)~""); + foreach( idx, ref v; m_array ) + if( auto ret = del(idx, v) ) + return ret; + return 0; + } + /// ditto + int opApply(int delegate(ref size_t idx, ref const Json obj) del) + const { + enforce(m_type == Type.Array, "opApply may only be called on arrays, not "~.to!string(m_type)~"."); + foreach( idx, ref v; m_array ) + if( auto ret = del(idx, v) ) + return ret; + return 0; + } + /// ditto + int opApply(int delegate(ref string idx, ref Json obj) del) + { + enforce(m_type == Type.Object, "opApply may only be called on objects, not "~.to!string(m_type)~"."); + foreach( idx, ref v; m_object ) + if( v.type != Type.Undefined ) + if( auto ret = del(idx, v) ) + return ret; + return 0; + } + /// ditto + int opApply(int delegate(ref string idx, ref const Json obj) del) + const { + enforce(m_type == Type.Object, "opApply may only be called on objects, not "~.to!string(m_type)~"."); + foreach( idx, ref v; m_object ) + if( v.type != Type.Undefined ) + if( auto ret = del(idx, v) ) + return ret; + return 0; + } + + /** + Converts the JSON value to the corresponding D type - types must match exactly. + */ + inout(T) opCast(T)() inout { return get!T; } + /// ditto + @property inout(T) get(T)() + inout { + checkType!T(); + static if( is(T == bool) ) return m_bool; + else static if( is(T == double) ) return m_float; + else static if( is(T == float) ) return cast(T)m_float; + else static if( is(T == long) ) return m_int; + else static if( is(T : long) ){ enforce(m_int <= T.max && m_int >= T.min); return cast(T)m_int; } + else static if( is(T == string) ) return m_string; + else static if( is(T == Json[]) ) return m_array; + else static if( is(T == Json[string]) ) return m_object; + else static assert("JSON can only be casted to (bool, long, double, string, Json[] or Json[string]. Not "~T.stringof~"."); + } + /// ditto + @property const(T) opt(T)(const(T) def = T.init) + const { + if( typeId!T != m_type ) return def; + return get!T; + } + /// ditto + @property T opt(T)(T def = T.init) + { + if( typeId!T != m_type ) return def; + return get!T; + } + + /** + Converts the JSON value to the corresponding D type - types are converted as neccessary. + */ + @property inout(T) to(T)() + inout { + static if( is(T == bool) ){ + final switch( m_type ){ + case Type.Undefined: return false; + case Type.Null: return false; + case Type.Bool: return m_bool; + case Type.Int: return m_int != 0; + case Type.Float: return m_float != 0; + case Type.String: return m_string.length > 0; + case Type.Array: return m_array.length > 0; + case Type.Object: return m_object.length > 0; + } + } else static if( is(T == double) ){ + final switch( m_type ){ + case Type.Undefined: return T.init; + case Type.Null: return 0; + case Type.Bool: return m_bool ? 1 : 0; + case Type.Int: return m_int; + case Type.Float: return m_float; + case Type.String: return .to!double(cast(string)m_string); + case Type.Array: return double.init; + case Type.Object: return double.init; + } + } else static if( is(T == float) ){ + final switch( m_type ){ + case Type.Undefined: return T.init; + case Type.Null: return 0; + case Type.Bool: return m_bool ? 1 : 0; + case Type.Int: return m_int; + case Type.Float: return m_float; + case Type.String: return .to!float(cast(string)m_string); + case Type.Array: return float.init; + case Type.Object: return float.init; + } + } + else static if( is(T == long) ){ + final switch( m_type ){ + case Type.Undefined: return 0; + case Type.Null: return 0; + case Type.Bool: return m_bool ? 1 : 0; + case Type.Int: return m_int; + case Type.Float: return cast(long)m_float; + case Type.String: return .to!long(m_string); + case Type.Array: return 0; + case Type.Object: return 0; + } + } else static if( is(T : long) ){ + final switch( m_type ){ + case Type.Undefined: return 0; + case Type.Null: return 0; + case Type.Bool: return m_bool ? 1 : 0; + case Type.Int: return cast(T)m_int; + case Type.Float: return cast(T)m_float; + case Type.String: return cast(T).to!long(cast(string)m_string); + case Type.Array: return 0; + case Type.Object: return 0; + } + } else static if( is(T == string) ){ + switch( m_type ){ + default: return toString(); + case Type.String: return m_string; + } + } else static if( is(T == Json[]) ){ + switch( m_type ){ + default: return Json([this]); + case Type.Array: return m_array; + } + } else static if( is(T == Json[string]) ){ + switch( m_type ){ + default: return Json(["value": this]); + case Type.Object: return m_object; + } + } else static assert("JSON can only be casted to (bool, long, double, string, Json[] or Json[string]. Not "~T.stringof~"."); + } + + /** + Performs unary operations on the JSON value. + + The following operations are supported for each type: + + $(DL + $(DT Null) $(DD none) + $(DT Bool) $(DD ~) + $(DT Int) $(DD +, -, ++, --) + $(DT Float) $(DD +, -, ++, --) + $(DT String) $(DD none) + $(DT Array) $(DD none) + $(DT Object) $(DD none) + ) + */ + Json opUnary(string op)() + const { + static if( op == "~" ){ + checkType!bool(); + return Json(~m_bool); + } else static if( op == "+" || op == "-" || op == "++" || op == "--" ){ + if( m_type == Type.Int ) mixin("return Json("~op~"m_int);"); + else if( m_type == Type.Float ) mixin("return Json("~op~"m_float);"); + else enforce(false, "'"~op~"' only allowed on scalar types, not on "~.to!string(m_type)~"."); + } else static assert("Unsupported operator '"~op~"' for type JSON."); + } + + /** + Performs binary operations between JSON values. + + The two JSON values must be of the same run time type or an exception + will be thrown. Only the operations listed are allowed for each of the + types. + + $(DL + $(DT Null) $(DD none) + $(DT Bool) $(DD &&, ||) + $(DT Int) $(DD +, -, *, /, %) + $(DT Float) $(DD +, -, *, /, %) + $(DT String) $(DD ~) + $(DT Array) $(DD ~) + $(DT Object) $(DD none) + ) + */ + Json opBinary(string op)(ref const(Json) other) + const { + enforce(m_type == other.m_type, "Binary operation '"~op~"' between "~.to!string(m_type)~" and "~.to!string(other.m_type)~" JSON objects."); + static if( op == "&&" ){ + enforce(m_type == Type.Bool, "'&&' only allowed for Type.Bool, not "~.to!string(m_type)~"."); + return Json(m_bool && other.m_bool); + } else static if( op == "||" ){ + enforce(m_type == Type.Bool, "'||' only allowed for Type.Bool, not "~.to!string(m_type)~"."); + return Json(m_bool || other.m_bool); + } else static if( op == "+" ){ + if( m_type == Type.Int ) return Json(m_int + other.m_int); + else if( m_type == Type.Float ) return Json(m_float + other.m_float); + else enforce(false, "'+' only allowed for scalar types, not "~.to!string(m_type)~"."); + } else static if( op == "-" ){ + if( m_type == Type.Int ) return Json(m_int - other.m_int); + else if( m_type == Type.Float ) return Json(m_float - other.m_float); + else enforce(false, "'-' only allowed for scalar types, not "~.to!string(m_type)~"."); + } else static if( op == "*" ){ + if( m_type == Type.Int ) return Json(m_int * other.m_int); + else if( m_type == Type.Float ) return Json(m_float * other.m_float); + else enforce(false, "'*' only allowed for scalar types, not "~.to!string(m_type)~"."); + } else static if( op == "/" ){ + if( m_type == Type.Int ) return Json(m_int / other.m_int); + else if( m_type == Type.Float ) return Json(m_float / other.m_float); + else enforce(false, "'/' only allowed for scalar types, not "~.to!string(m_type)~"."); + } else static if( op == "%" ){ + if( m_type == Type.Int ) return Json(m_int % other.m_int); + else if( m_type == Type.Float ) return Json(m_float % other.m_float); + else enforce(false, "'%' only allowed for scalar types, not "~.to!string(m_type)~"."); + } else static if( op == "~" ){ + if( m_type == Type.String ) return Json(m_string ~ other.m_string); + else enforce(false, "'~' only allowed for strings, not "~.to!string(m_type)~"."); + } else static assert("Unsupported operator '"~op~"' for type JSON."); + assert(false); + } + /// ditto + Json opBinary(string op)(Json other) + if( op == "~" ) + { + static if( op == "~" ){ + if( m_type == Type.String ) return Json(m_string ~ other.m_string); + else if( m_type == Type.Array ) return Json(m_array ~ other.m_array); + else enforce(false, "'~' only allowed for strings and arrays, not "~.to!string(m_type)~"."); + } else static assert("Unsupported operator '"~op~"' for type JSON."); + assert(false); + } + /// ditto + void opOpAssign(string op)(Json other) + if( op == "+" || op == "-" || op == "*" ||op == "/" || op == "%" ) + { + enforce(m_type == other.m_type, "Binary operation '"~op~"' between "~.to!string(m_type)~" and "~.to!string(other.m_type)~" JSON objects."); + static if( op == "+" ){ + if( m_type == Type.Int ) m_int += other.m_int; + else if( m_type == Type.Float ) m_float += other.m_float; + else enforce(false, "'+' only allowed for scalar types, not "~.to!string(m_type)~"."); + } else static if( op == "-" ){ + if( m_type == Type.Int ) m_int -= other.m_int; + else if( m_type == Type.Float ) m_float -= other.m_float; + else enforce(false, "'-' only allowed for scalar types, not "~.to!string(m_type)~"."); + } else static if( op == "*" ){ + if( m_type == Type.Int ) m_int *= other.m_int; + else if( m_type == Type.Float ) m_float *= other.m_float; + else enforce(false, "'*' only allowed for scalar types, not "~.to!string(m_type)~"."); + } else static if( op == "/" ){ + if( m_type == Type.Int ) m_int /= other.m_int; + else if( m_type == Type.Float ) m_float /= other.m_float; + else enforce(false, "'/' only allowed for scalar types, not "~.to!string(m_type)~"."); + } else static if( op == "%" ){ + if( m_type == Type.Int ) m_int %= other.m_int; + else if( m_type == Type.Float ) m_float %= other.m_float; + else enforce(false, "'%' only allowed for scalar types, not "~.to!string(m_type)~"."); + } /*else static if( op == "~" ){ + if( m_type == Type.String ) m_string ~= other.m_string; + else if( m_type == Type.Array ) m_array ~= other.m_array; + else enforce(false, "'%' only allowed for scalar types, not "~.to!string(m_type)~"."); + }*/ else static assert("Unsupported operator '"~op~"' for type JSON."); + assert(false); + } + /// ditto + Json opBinary(string op)(bool other) const { checkType!bool(); mixin("return Json(m_bool "~op~" other);"); } + /// ditto + Json opBinary(string op)(long other) const { checkType!long(); mixin("return Json(m_int "~op~" other);"); } + /// ditto + Json opBinary(string op)(double other) const { checkType!double(); mixin("return Json(m_float "~op~" other);"); } + /// ditto + Json opBinary(string op)(string other) const { checkType!string(); mixin("return Json(m_string "~op~" other);"); } + /// ditto + Json opBinary(string op)(Json[] other) { checkType!(Json[])(); mixin("return Json(m_array "~op~" other);"); } + /// ditto + Json opBinaryRight(string op)(bool other) const { checkType!bool(); mixin("return Json(other "~op~" m_bool);"); } + /// ditto + Json opBinaryRight(string op)(long other) const { checkType!long(); mixin("return Json(other "~op~" m_int);"); } + /// ditto + Json opBinaryRight(string op)(double other) const { checkType!double(); mixin("return Json(other "~op~" m_float);"); } + /// ditto + Json opBinaryRight(string op)(string other) const if(op == "~") { checkType!string(); return Json(other ~ m_string); } + /// ditto + inout(Json)* opBinaryRight(string op)(string other) inout if(op == "in") { + checkType!(Json[string])(); + auto pv = other in m_object; + if( !pv ) return null; + if( pv.type == Type.Undefined ) return null; + return pv; + } + /// ditto + Json opBinaryRight(string op)(Json[] other) { checkType!(Json[])(); mixin("return Json(other "~op~" m_array);"); } + + /** + Allows to access existing fields of a JSON object using dot syntax. + */ + @property const(Json) opDispatch(string prop)() const { return opIndex(prop); } + /// ditto + @property ref Json opDispatch(string prop)() { return opIndex(prop); } + + /** + Compares two JSON values for equality. + + If the two values have different types, they are considered unequal. + This differs with ECMA script, which performs a type conversion before + comparing the values. + */ + bool opEquals(ref const Json other) + const { + if( m_type != other.m_type ) return false; + final switch(m_type){ + case Type.Undefined: return false; + case Type.Null: return true; + case Type.Bool: return m_bool == other.m_bool; + case Type.Int: return m_int == other.m_int; + case Type.Float: return m_float == other.m_float; + case Type.String: return m_string == other.m_string; + case Type.Array: return m_array == other.m_array; + case Type.Object: return m_object == other.m_object; + } + } + /// ditto + bool opEquals(const Json other) const { return opEquals(other); } + /// ditto + bool opEquals(typeof(null)) const { return m_type == Type.Null; } + /// ditto + bool opEquals(bool v) const { return m_type == Type.Bool && m_bool == v; } + /// ditto + bool opEquals(long v) const { return m_type == Type.Int && m_int == v; } + /// ditto + bool opEquals(double v) const { return m_type == Type.Float && m_float == v; } + /// ditto + bool opEquals(string v) const { return m_type == Type.String && m_string == v; } + + /** + Compares two JSON values. + + If the types of the two values differ, the value with the smaller type + id is considered the smaller value. This differs from ECMA script, which + performs a type conversion before comparing the values. + + JSON values of type Object cannot be compared and will throw an + exception. + */ + int opCmp(ref const Json other) + const { + if( m_type != other.m_type ) return m_type < other.m_type ? -1 : 1; + final switch(m_type){ + case Type.Undefined: return 0; + case Type.Null: return 0; + case Type.Bool: return m_bool < other.m_bool ? -1 : m_bool == other.m_bool ? 0 : 1; + case Type.Int: return m_int < other.m_int ? -1 : m_int == other.m_int ? 0 : 1; + case Type.Float: return m_float < other.m_float ? -1 : m_float == other.m_float ? 0 : 1; + case Type.String: return m_string < other.m_string ? -1 : m_string == other.m_string ? 0 : 1; + case Type.Array: return m_array < other.m_array ? -1 : m_array == other.m_array ? 0 : 1; + case Type.Object: + enforce(false, "JSON objects cannot be compared."); + assert(false); + } + } + + + + /** + Returns the type id corresponding to the given D type. + */ + static @property Type typeId(T)() { + static if( is(T == typeof(null)) ) return Type.Null; + else static if( is(T == bool) ) return Type.Bool; + else static if( is(T == double) ) return Type.Float; + else static if( is(T == float) ) return Type.Float; + else static if( is(T : long) ) return Type.Int; + else static if( is(T == string) ) return Type.String; + else static if( is(T == Json[]) ) return Type.Array; + else static if( is(T == Json[string]) ) return Type.Object; + else static assert(false, "Unsupported JSON type '"~T.stringof~"'. Only bool, long, double, string, Json[] and Json[string] are allowed."); + } + + /** + Returns the JSON object as a string. + + For large JSON values use writeJsonString instead as this function will store the whole string + in memory, whereas writeJsonString writes it out bit for bit. + + See_Also: writeJsonString, toPrettyString + */ + string toString() + const { + auto ret = appender!string(); + writeJsonString(ret, this); + return ret.data; + } + + /** + Returns the JSON object as a "pretty" string. + + --- + auto json = Json(["foo": Json("bar")]); + writeln(json.toPrettyString()); + + // output: + // { + // "foo": "bar" + // } + --- + + Params: + level = Specifies the base amount of indentation for the output. Indentation is always + done using tab characters. + + See_Also: writePrettyJsonString, toString + */ + string toPrettyString(int level = 0) + const { + auto ret = appender!string(); + writePrettyJsonString(ret, this, level); + return ret.data; + } + + private void checkType(T)() + const { + string dbg; + if( m_type == Type.Undefined ) dbg = " field "~m_string; + enforce(typeId!T == m_type, "Trying to access JSON"~dbg~" of type "~.to!string(m_type)~" as "~T.stringof~"."); + } + + /*invariant() + { + assert(m_type >= Type.Undefined && m_type <= Type.Object); + }*/ +} + + +/******************************************************************************/ +/* public functions */ +/******************************************************************************/ + +/** + Parses the given range as a JSON string and returns the corresponding Json object. + + The range is shrunk during parsing, leaving any remaining text that is not part of + the JSON contents. + + Throws an Exception if any parsing error occured. +*/ +Json parseJson(R)(ref R range, int* line = null) + if( is(R == string) ) +{ + Json ret; + enforce(!range.empty, "JSON string is empty."); + + skipWhitespace(range, line); + + version(JsonLineNumbers){ + import vibecompat.core.log; + int curline = line ? *line : 0; + scope(failure) logError("Error in line: %d", curline); + } + + switch( range.front ){ + case 'f': + enforce(range[1 .. $].startsWith("alse"), "Expected 'false', got '"~range[0 .. 5]~"'."); + range.popFrontN(5); + ret = false; + break; + case 'n': + enforce(range[1 .. $].startsWith("ull"), "Expected 'null', got '"~range[0 .. 4]~"'."); + range.popFrontN(4); + ret = null; + break; + case 't': + enforce(range[1 .. $].startsWith("rue"), "Expected 'true', got '"~range[0 .. 4]~"'."); + range.popFrontN(4); + ret = true; + break; + case '0': .. case '9'+1: + case '-': + bool is_float; + auto num = skipNumber(range, is_float); + if( is_float ) ret = to!double(num); + else ret = to!long(num); + break; + case '\"': + ret = skipJsonString(range); + break; + case '[': + Json[] arr; + range.popFront(); + while(true) { + skipWhitespace(range, line); + enforce(!range.empty); + if(range.front == ']') break; + arr ~= parseJson(range, line); + skipWhitespace(range, line); + enforce(!range.empty && (range.front == ',' || range.front == ']'), "Expected ']' or ','."); + if( range.front == ']' ) break; + else range.popFront(); + } + range.popFront(); + ret = arr; + break; + case '{': + Json[string] obj; + range.popFront(); + while(true) { + skipWhitespace(range, line); + enforce(!range.empty); + if(range.front == '}') break; + string key = skipJsonString(range); + skipWhitespace(range, line); + enforce(range.startsWith(":"), "Expected ':' for key '" ~ key ~ "'"); + range.popFront(); + skipWhitespace(range, line); + Json itm = parseJson(range, line); + obj[key] = itm; + skipWhitespace(range, line); + enforce(!range.empty && (range.front == ',' || range.front == '}'), "Expected '}' or ',' - got '"~range[0]~"'."); + if( range.front == '}' ) break; + else range.popFront(); + } + range.popFront(); + ret = obj; + break; + default: + enforce(false, "Expected valid json token, got '"~to!string(range.length)~range[0 .. range.length>12?12:range.length]~"'."); + } + + assert(ret.type != Json.Type.Undefined); + version(JsonLineNumbers) ret.line = curline; + return ret; +} + +/** + Parses the given JSON string and returns the corresponding Json object. + + Throws an Exception if any parsing error occurs. +*/ +Json parseJsonString(string str) +{ + auto ret = parseJson(str); + enforce(str.strip().length == 0, "Expected end of string after JSON value."); + return ret; +} + +unittest { + assert(parseJsonString("null") == Json(null)); + assert(parseJsonString("true") == Json(true)); + assert(parseJsonString("false") == Json(false)); + assert(parseJsonString("1") == Json(1)); + assert(parseJsonString("2.0") == Json(2.0)); + assert(parseJsonString("\"test\"") == Json("test")); + assert(parseJsonString("[1, 2, 3]") == Json([Json(1), Json(2), Json(3)])); + assert(parseJsonString("{\"a\": 1}") == Json(["a": Json(1)])); + assert(parseJsonString(`"\\\/\b\f\n\r\t\u1234"`).get!string == "\\/\b\f\n\r\t\u1234"); +} + + +/** + Serializes the given value to JSON. + + The following types of values are supported: + + $(DL + $(DT Json) $(DD Used as-is) + $(DT null) $(DD Converted to Json.Type.Null) + $(DT bool) $(DD Converted to Json.Type.Bool) + $(DT float, double) $(DD Converted to Json.Type.Double) + $(DT short, ushort, int, uint, long, ulong) $(DD Converted to Json.Type.Int) + $(DT string) $(DD Converted to Json.Type.String) + $(DT T[]) $(DD Converted to Json.Type.Array) + $(DT T[string]) $(DD Converted to Json.Type.Object) + $(DT struct) $(DD Converted to Json.Type.Object) + $(DT class) $(DD Converted to Json.Type.Object or Json.Type.Null) + ) + + All entries of an array or an associative array, as well as all R/W properties and + all public fields of a struct/class are recursively serialized using the same rules. + + Fields ending with an underscore will have the last underscore stripped in the + serialized output. This makes it possible to use fields with D keywords as their name + by simply appending an underscore. + + The following methods can be used to customize the serialization of structs/classes: + + --- + Json toJson() const; + static T fromJson(Json src); + + string toString() const; + static T fromString(string src); + --- + + The methods will have to be defined in pairs. The first pair that is implemented by + the type will be used for serialization (i.e. toJson overrides toString). +*/ +Json serializeToJson(T)(T value) +{ + alias Unqual!T TU; + static if( is(TU == Json) ) return value; + else static if( is(TU == typeof(null)) ) return Json(null); + else static if( is(TU == bool) ) return Json(value); + else static if( is(TU == float) ) return Json(cast(double)value); + else static if( is(TU == double) ) return Json(value); + else static if( is(TU == DateTime) ) return Json(value.toISOExtString()); + else static if( is(TU == SysTime) ) return Json(value.toISOExtString()); + else static if( is(TU : long) ) return Json(cast(long)value); + else static if( is(TU == string) ) return Json(value); + else static if( isArray!T ){ + auto ret = new Json[value.length]; + foreach( i; 0 .. value.length ) + ret[i] = serializeToJson(value[i]); + return Json(ret); + } else static if( isAssociativeArray!TU ){ + Json[string] ret; + foreach( string key, value; value ) + ret[key] = serializeToJson(value); + return Json(ret); + } else static if( __traits(compiles, value = T.fromJson(value.toJson())) ){ + return value.toJson(); + } else static if( __traits(compiles, value = T.fromString(value.toString())) ){ + return Json(value.toString()); + } else static if( is(TU == struct) ){ + Json[string] ret; + foreach( m; __traits(allMembers, T) ){ + static if( isRWField!(TU, m) ){ + auto mv = __traits(getMember, value, m); + ret[underscoreStrip(m)] = serializeToJson(mv); + } + } + return Json(ret); + } else static if( is(TU == class) ){ + if( value is null ) return Json(null); + Json[string] ret; + foreach( m; __traits(allMembers, T) ){ + static if( isRWField!(TU, m) ){ + auto mv = __traits(getMember, value, m); + ret[underscoreStrip(m)] = serializeToJson(mv); + } + } + return Json(ret); + } else static if( isPointer!TU ){ + if( value is null ) return Json(null); + return serializeToJson(*value); + } else { + static assert(false, "Unsupported type '"~T.stringof~"' for JSON serialization."); + } +} + + +/** + Deserializes a JSON value into the destination variable. + + The same types as for serializeToJson() are supported and handled inversely. +*/ +void deserializeJson(T)(ref T dst, Json src) +{ + dst = deserializeJson!T(src); +} +/// ditto +T deserializeJson(T)(Json src) +{ + static if( is(T == Json) ) return src; + else static if( is(T == typeof(null)) ){ return null; } + else static if( is(T == bool) ) return src.get!bool; + else static if( is(T == float) ) return src.to!float; // since doubles are frequently serialized without + else static if( is(T == double) ) return src.to!double; // a decimal point, we allow conversions here + else static if( is(T == DateTime) ) return DateTime.fromISOExtString(src.get!string); + else static if( is(T == SysTime) ) return SysTime.fromISOExtString(src.get!string); + else static if( is(T : long) ) return cast(T)src.get!long; + else static if( is(T == string) ) return src.get!string; + else static if( isArray!T ){ + alias typeof(T.init[0]) TV; + auto dst = new Unqual!TV[src.length]; + foreach( size_t i, v; src ) + dst[i] = deserializeJson!(Unqual!TV)(v); + return dst; + } else static if( isAssociativeArray!T ){ + alias typeof(T.init.values[0]) TV; + Unqual!TV[string] dst; + foreach( string key, value; src ) + dst[key] = deserializeJson!(Unqual!TV)(value); + return dst; + } else static if( __traits(compiles, { T dst; dst = T.fromJson(dst.toJson()); }()) ){ + return T.fromJson(src); + } else static if( __traits(compiles, { T dst; dst = T.fromString(dst.toString()); }()) ){ + return T.fromString(src.get!string); + } else static if( is(T == struct) ){ + T dst; + foreach( m; __traits(allMembers, T) ){ + static if( isRWPlainField!(T, m) || isRWField!(T, m) ){ + alias typeof(__traits(getMember, dst, m)) TM; + __traits(getMember, dst, m) = deserializeJson!TM(src[underscoreStrip(m)]); + } + } + return dst; + } else static if( is(T == class) ){ + if( src.type == Json.Type.Null ) return null; + auto dst = new T; + foreach( m; __traits(allMembers, T) ){ + static if( isRWPlainField!(T, m) || isRWField!(T, m) ){ + alias typeof(__traits(getMember, dst, m)) TM; + __traits(getMember, dst, m) = deserializeJson!TM(src[underscoreStrip(m)]); + } + } + return dst; + } else static if( isPointer!T ){ + if( src.type == Json.Type.Null ) return null; + alias typeof(*T.init) TD; + dst = new TD; + *dst = deserializeJson!TD(src); + return dst; + } else { + static assert(false, "Unsupported type '"~T.stringof~"' for JSON serialization."); + } +} + +unittest { + import std.stdio; + static struct S { float a; double b; bool c; int d; string e; byte f; ubyte g; long h; ulong i; float[] j; } + immutable S t = {1.5, -3.0, true, int.min, "Test", -128, 255, long.min, ulong.max, [1.1, 1.2, 1.3]}; + S u; + deserializeJson(u, serializeToJson(t)); + assert(t.a == u.a); + assert(t.b == u.b); + assert(t.c == u.c); + assert(t.d == u.d); + assert(t.e == u.e); + assert(t.f == u.f); + assert(t.g == u.g); + assert(t.h == u.h); + assert(t.i == u.i); + assert(t.j == u.j); +} + +unittest { + static class C { + int a; + private int _b; + @property int b() const { return _b; } + @property void b(int v) { _b = v; } + + @property int test() const { return 10; } + + void test2() {} + } + C c = new C; + c.a = 1; + c.b = 2; + + C d; + deserializeJson(d, serializeToJson(c)); + assert(c.a == d.a); + assert(c.b == d.b); +} + + +/** + Writes the given JSON object as a JSON string into the destination range. + + This function will convert the given JSON value to a string without adding + any white space between tokens (no newlines, no indentation and no padding). + The output size is thus minizized, at the cost of bad human readability. + + Params: + dst = References the string output range to which the result is written. + json = Specifies the JSON value that is to be stringified. + + See_Also: Json.toString, writePrettyJsonString +*/ +void writeJsonString(R)(ref R dst, in Json json) +// if( isOutputRange!R && is(ElementEncodingType!R == char) ) +{ + final switch( json.type ){ + case Json.Type.Undefined: dst.put("undefined"); break; + case Json.Type.Null: dst.put("null"); break; + case Json.Type.Bool: dst.put(cast(bool)json ? "true" : "false"); break; + case Json.Type.Int: formattedWrite(dst, "%d", json.get!long); break; + case Json.Type.Float: formattedWrite(dst, "%.16g", json.get!double); break; + case Json.Type.String: + dst.put("\""); + jsonEscape(dst, cast(string)json); + dst.put("\""); + break; + case Json.Type.Array: + dst.put("["); + bool first = true; + foreach( ref const Json e; json ){ + if( e.type == Json.Type.Undefined ) continue; + if( !first ) dst.put(","); + first = false; + writeJsonString(dst, e); + } + dst.put("]"); + break; + case Json.Type.Object: + dst.put("{"); + bool first = true; + foreach( string k, ref const Json e; json ){ + if( e.type == Json.Type.Undefined ) continue; + if( !first ) dst.put(","); + first = false; + dst.put("\""); + jsonEscape(dst, k); + dst.put("\":"); + writeJsonString(dst, e); + } + dst.put("}"); + break; + } +} + +/** + Writes the given JSON object as a prettified JSON string into the destination range. + + The output will contain newlines and indents to make the output human readable. + + Params: + dst = References the string output range to which the result is written. + json = Specifies the JSON value that is to be stringified. + level = Specifies the base amount of indentation for the output. Indentation is always + done using tab characters. + + See_Also: Json.toPrettyString, writeJsonString +*/ +void writePrettyJsonString(R)(ref R dst, in Json json, int level = 0) +// if( isOutputRange!R && is(ElementEncodingType!R == char) ) +{ + final switch( json.type ){ + case Json.Type.Undefined: dst.put("undefined"); break; + case Json.Type.Null: dst.put("null"); break; + case Json.Type.Bool: dst.put(cast(bool)json ? "true" : "false"); break; + case Json.Type.Int: formattedWrite(dst, "%d", json.get!long); break; + case Json.Type.Float: formattedWrite(dst, "%.16g", json.get!double); break; + case Json.Type.String: + dst.put("\""); + jsonEscape(dst, cast(string)json); + dst.put("\""); + break; + case Json.Type.Array: + dst.put("["); + bool first = true; + foreach( e; json ){ + if( e.type == Json.Type.Undefined ) continue; + if( !first ) dst.put(","); + first = false; + dst.put("\n"); + foreach( tab; 0 .. level ) dst.put('\t'); + writePrettyJsonString(dst, e, level+1); + } + if( json.length > 0 ) { + dst.put('\n'); + foreach( tab; 0 .. (level-1) ) dst.put('\t'); + } + dst.put("]"); + break; + case Json.Type.Object: + dst.put("{"); + bool first = true; + foreach( string k, e; json ){ + if( e.type == Json.Type.Undefined ) continue; + if( !first ) dst.put(","); + dst.put("\n"); + first = false; + foreach( tab; 0 .. level ) dst.put('\t'); + dst.put("\""); + jsonEscape(dst, k); + dst.put("\": "); + writePrettyJsonString(dst, e, level+1); + } + if( json.length > 0 ) { + dst.put('\n'); + foreach( tab; 0 .. (level-1) ) dst.put('\t'); + } + dst.put("}"); + break; + } +} + + +/** Deprecated aliases for backwards compatibility. + + Use writeJsonString and writePrettyJsonString instead. +*/ +deprecated("Please use writeJsonString instead.") alias writeJsonString toJson; +/// +deprecated("Please use writePrettyJsonString instead.") alias writePrettyJsonString toPrettyJson; + + +/// private +private void jsonEscape(R)(ref R dst, string s) +{ + foreach( ch; s ){ + switch(ch){ + default: dst.put(ch); break; + case '\\': dst.put("\\\\"); break; + case '\r': dst.put("\\r"); break; + case '\n': dst.put("\\n"); break; + case '\t': dst.put("\\t"); break; + case '\"': dst.put("\\\""); break; + } + } +} + +/// private +private string jsonUnescape(R)(ref R range) +{ + auto ret = appender!string(); + while(!range.empty){ + auto ch = range.front; + switch( ch ){ + case '"': return ret.data; + case '\\': + range.popFront(); + enforce(!range.empty, "Unterminated string escape sequence."); + switch(range.front){ + default: enforce("Invalid string escape sequence."); break; + case '"': ret.put('\"'); range.popFront(); break; + case '\\': ret.put('\\'); range.popFront(); break; + case '/': ret.put('/'); range.popFront(); break; + case 'b': ret.put('\b'); range.popFront(); break; + case 'f': ret.put('\f'); range.popFront(); break; + case 'n': ret.put('\n'); range.popFront(); break; + case 'r': ret.put('\r'); range.popFront(); break; + case 't': ret.put('\t'); range.popFront(); break; + case 'u': + range.popFront(); + dchar uch = 0; + foreach( i; 0 .. 4 ){ + uch *= 16; + enforce(!range.empty, "Unicode sequence must be '\\uXXXX'."); + auto dc = range.front; + range.popFront(); + if( dc >= '0' && dc <= '9' ) uch += dc - '0'; + else if( dc >= 'a' && dc <= 'f' ) uch += dc - 'a' + 10; + else if( dc >= 'A' && dc <= 'F' ) uch += dc - 'A' + 10; + else enforce(false, "Unicode sequence must be '\\uXXXX'."); + } + ret.put(uch); + break; + } + break; + default: + ret.put(ch); + range.popFront(); + break; + } + } + return ret.data; +} + +private string skipNumber(ref string s, out bool is_float) +{ + size_t idx = 0; + is_float = false; + if( s[idx] == '-' ) idx++; + if( s[idx] == '0' ) idx++; + else { + enforce(isDigit(s[idx++]), "Digit expected at beginning of number."); + while( idx < s.length && isDigit(s[idx]) ) idx++; + } + + if( idx < s.length && s[idx] == '.' ){ + idx++; + is_float = true; + while( idx < s.length && isDigit(s[idx]) ) idx++; + } + + if( idx < s.length && (s[idx] == 'e' || s[idx] == 'E') ){ + idx++; + is_float = true; + if( idx < s.length && (s[idx] == '+' || s[idx] == '-') ) idx++; + enforce( idx < s.length && isDigit(s[idx]), "Expected exponent." ~ s[0 .. idx]); + idx++; + while( idx < s.length && isDigit(s[idx]) ) idx++; + } + + string ret = s[0 .. idx]; + s = s[idx .. $]; + return ret; +} + +private string skipJsonString(ref string s, int* line = null) +{ + enforce(s.length >= 2 && s[0] == '\"', "too small: '" ~ s ~ "'"); + s = s[1 .. $]; + string ret = jsonUnescape(s); + enforce(s.length > 0 && s[0] == '\"', "Unterminated string literal."); + s = s[1 .. $]; + return ret; +} + +private void skipWhitespace(ref string s, int* line = null) +{ + while( s.length > 0 ){ + switch( s[0] ){ + default: return; + case ' ', '\t': s = s[1 .. $]; break; + case '\n': + s = s[1 .. $]; + if( s.length > 0 && s[0] == '\r' ) s = s[1 .. $]; + if( line ) (*line)++; + break; + case '\r': + s = s[1 .. $]; + if( s.length > 0 && s[0] == '\n' ) s = s[1 .. $]; + if( line ) (*line)++; + break; + } + } +} + +/// private +private bool isDigit(T)(T ch){ return ch >= '0' && ch <= '9'; } + +private string underscoreStrip(string field_name) +{ + if( field_name.length < 1 || field_name[$-1] != '_' ) return field_name; + else return field_name[0 .. $-1]; +} diff --git a/source/dub/internal/vibecompat/data/utils.d b/source/dub/internal/vibecompat/data/utils.d new file mode 100644 index 0000000..c39df38 --- /dev/null +++ b/source/dub/internal/vibecompat/data/utils.d @@ -0,0 +1,30 @@ +/** + Utility functions for data serialization + + Copyright: © 2012 RejectedSoftware e.K. + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Sönke Ludwig +*/ +module dub.internal.vibecompat.data.utils; + +public import std.traits; + + +template isRWPlainField(T, string M) +{ + static if( !__traits(compiles, typeof(__traits(getMember, T, M))) ){ + enum isRWPlainField = false; + } else { + //pragma(msg, T.stringof~"."~M~":"~typeof(__traits(getMember, T, M)).stringof); + enum isRWPlainField = isRWField!(T, M) && __traits(compiles, *(&__traits(getMember, Tgen!T(), M)) = *(&__traits(getMember, Tgen!T(), M))); + } +} + +template isRWField(T, string M) +{ + enum isRWField = __traits(compiles, __traits(getMember, Tgen!T(), M) = __traits(getMember, Tgen!T(), M)); + //pragma(msg, T.stringof~"."~M~": "~(isRWField?"1":"0")); +} + +/// private +private T Tgen(T)(){ return T.init; } diff --git a/source/dub/internal/vibecompat/inet/path.d b/source/dub/internal/vibecompat/inet/path.d new file mode 100644 index 0000000..4a662f9 --- /dev/null +++ b/source/dub/internal/vibecompat/inet/path.d @@ -0,0 +1,303 @@ +/** + Contains routines for high level path handling. + + Copyright: © 2012 RejectedSoftware e.K. + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Sönke Ludwig +*/ +module dub.internal.vibecompat.inet.path; + +import std.algorithm; +import std.array; +import std.conv; +import std.exception; +import std.string; + + +/** + Represents an absolute or relative file system path. + + This struct allows to do safe operations on paths, such as concatenation and sub paths. Checks + are done to disallow invalid operations such as concatenating two absolute paths. It also + validates path strings and allows for easy checking of malicious relative paths. +*/ +struct Path { + private { + immutable(PathEntry)[] m_nodes; + bool m_absolute = false; + bool m_endsWithSlash = false; + } + + /// Constructs a Path object by parsing a path string. + this(string pathstr) + { + m_nodes = cast(immutable)splitPath(pathstr); + m_absolute = (pathstr.startsWith("/") || m_nodes.length > 0 && m_nodes[0].toString().countUntil(':')>0); + m_endsWithSlash = pathstr.endsWith("/"); + foreach( e; m_nodes ) assert(e.toString().length > 0); + } + + /// Constructs a path object from a list of PathEntry objects. + this(immutable(PathEntry)[] nodes, bool absolute) + { + m_nodes = nodes; + m_absolute = absolute; + } + + /// Constructs a relative path with one path entry. + this(PathEntry entry){ + m_nodes = [entry]; + m_absolute = false; + } + + /// Determines if the path is absolute. + @property bool absolute() const { return m_absolute; } + + /// Resolves all '.' and '..' path entries as far as possible. + void normalize() + { + immutable(PathEntry)[] newnodes; + foreach( n; m_nodes ){ + switch(n.toString()){ + default: + newnodes ~= n; + break; + case ".": break; + case "..": + enforce(!m_absolute || newnodes.length > 0, "Path goes below root node."); + if( newnodes.length > 0 && newnodes[$-1] != ".." ) newnodes = newnodes[0 .. $-1]; + else newnodes ~= n; + break; + } + } + m_nodes = newnodes; + } + + /// Converts the Path back to a string representation using slashes. + string toString() + const { + if( m_nodes.empty ) return absolute ? "/" : ""; + + Appender!string ret; + + // for absolute paths start with / + if( absolute ) ret.put('/'); + + foreach( i, f; m_nodes ){ + if( i > 0 ) ret.put('/'); + ret.put(f.toString()); + } + + if( m_nodes.length > 0 && m_endsWithSlash ) + ret.put('/'); + + return ret.data; + } + + /// Converts the Path object to a native path string (backslash as path separator on Windows). + string toNativeString() + const { + Appender!string ret; + + // for absolute unix paths start with / + version(Posix) { if(absolute) ret.put('/'); } + + foreach( i, f; m_nodes ){ + version(Windows) { if( i > 0 ) ret.put('\\'); } + version(Posix) { if( i > 0 ) ret.put('/'); } + else { enforce("Unsupported OS"); } + ret.put(f.toString()); + } + + if( m_nodes.length > 0 && m_endsWithSlash ){ + version(Windows) { ret.put('\\'); } + version(Posix) { ret.put('/'); } + } + + return ret.data; + } + + /// Tests if `rhs` is an anchestor or the same as this path. + bool startsWith(const Path rhs) const { + if( rhs.m_nodes.length > m_nodes.length ) return false; + foreach( i; 0 .. rhs.m_nodes.length ) + if( m_nodes[i] != rhs.m_nodes[i] ) + return false; + return true; + } + + /// Computes the relative path from `parentPath` to this path. + Path relativeTo(const Path parentPath) const { + version(Windows){ + // a path such as ..\C:\windows is not valid, so force the path to stay absolute in this case + if( this.absolute && !this.empty && m_nodes[0].toString().endsWith(":") && + !parentPath.startsWith(this[0 .. 1]) ) + { + return this; + } + } + int nup = 0; + while( parentPath.length > nup && !startsWith(parentPath[0 .. parentPath.length-nup]) ){ + nup++; + } + Path ret = Path(null, false); + ret.m_endsWithSlash = true; + foreach( i; 0 .. nup ) ret ~= ".."; + ret ~= Path(m_nodes[parentPath.length-nup .. $], false); + return ret; + } + + /// The last entry of the path + @property ref immutable(PathEntry) head() const { enforce(m_nodes.length > 0); return m_nodes[$-1]; } + + /// The parent path + @property Path parentPath() const { return this[0 .. length-1]; } + + /// The ist of path entries of which this path is composed + @property immutable(PathEntry)[] nodes() const { return m_nodes; } + + /// The number of path entries of which this path is composed + @property size_t length() const { return m_nodes.length; } + + /// True if the path contains no entries + @property bool empty() const { return m_nodes.length == 0; } + + /// Determines if the path ends with a slash (i.e. is a directory) + @property bool endsWithSlash() const { return m_endsWithSlash; } + /// ditto + @property void endsWithSlash(bool v) { m_endsWithSlash = v; } + + /// Determines if this path goes outside of its base path (i.e. begins with '..'). + @property bool external() const { return !m_absolute && m_nodes.length > 0 && m_nodes[0].m_name == ".."; } + + ref immutable(PathEntry) opIndex(size_t idx) const { return m_nodes[idx]; } + Path opSlice(size_t start, size_t end) const { + auto ret = Path(m_nodes[start .. end], start == 0 ? absolute : false); + if( end == m_nodes.length ) ret.m_endsWithSlash = m_endsWithSlash; + return ret; + } + size_t opDollar(int dim)() const if(dim == 0) { return m_nodes.length; } + + + Path opBinary(string OP)(const Path rhs) const if( OP == "~" ) { + Path ret; + ret.m_nodes = m_nodes; + ret.m_absolute = m_absolute; + ret.m_endsWithSlash = rhs.m_endsWithSlash; + ret.normalize(); // needed to avoid "."~".." become "" instead of ".." + + assert(!rhs.absolute, "Trying to append absolute path."); + size_t idx = m_nodes.length; + foreach(folder; rhs.m_nodes){ + switch(folder.toString()){ + default: ret.m_nodes = ret.m_nodes ~ folder; break; + case ".": break; + case "..": + enforce(!ret.absolute || ret.m_nodes.length > 0, "Relative path goes below root node!"); + if( ret.m_nodes.length > 0 && ret.m_nodes[$-1].toString() != ".." ) + ret.m_nodes = ret.m_nodes[0 .. $-1]; + else ret.m_nodes = ret.m_nodes ~ folder; + break; + } + } + return ret; + } + + Path opBinary(string OP)(string rhs) const if( OP == "~" ) { assert(rhs.length > 0); return opBinary!"~"(Path(rhs)); } + Path opBinary(string OP)(PathEntry rhs) const if( OP == "~" ) { assert(rhs.toString().length > 0); return opBinary!"~"(Path(rhs)); } + void opOpAssign(string OP)(string rhs) if( OP == "~" ) { assert(rhs.length > 0); opOpAssign!"~"(Path(rhs)); } + void opOpAssign(string OP)(PathEntry rhs) if( OP == "~" ) { assert(rhs.toString().length > 0); opOpAssign!"~"(Path(rhs)); } + void opOpAssign(string OP)(Path rhs) if( OP == "~" ) { auto p = this ~ rhs; m_nodes = p.m_nodes; m_endsWithSlash = rhs.m_endsWithSlash; } + + /// Tests two paths for equality using '=='. + bool opEquals(ref const Path rhs) const { + if( m_absolute != rhs.m_absolute ) return false; + if( m_endsWithSlash != rhs.m_endsWithSlash ) return false; + if( m_nodes.length != rhs.length ) return false; + foreach( i; 0 .. m_nodes.length ) + if( m_nodes[i] != rhs.m_nodes[i] ) + return false; + return true; + } + /// ditto + bool opEquals(const Path other) const { return opEquals(other); } + + int opCmp(ref const Path rhs) const { + if( m_absolute != rhs.m_absolute ) return cast(int)m_absolute - cast(int)rhs.m_absolute; + foreach( i; 0 .. min(m_nodes.length, rhs.m_nodes.length) ) + if( m_nodes[i] != rhs.m_nodes[i] ) + return m_nodes[i].opCmp(rhs.m_nodes[i]); + if( m_nodes.length > rhs.m_nodes.length ) return 1; + if( m_nodes.length < rhs.m_nodes.length ) return -1; + return 0; + } +} + +struct PathEntry { + private { + string m_name; + } + + this(string str) + { + assert(str.countUntil('/') < 0 && str.countUntil('\\') < 0); + m_name = str; + } + + string toString() const { return m_name; } + + Path opBinary(string OP)(PathEntry rhs) const if( OP == "~" ) { return Path(cast(immutable)[this, rhs], false); } + + bool opEquals(ref const PathEntry rhs) const { return m_name == rhs.m_name; } + bool opEquals(PathEntry rhs) const { return m_name == rhs.m_name; } + bool opEquals(string rhs) const { return m_name == rhs; } + int opCmp(ref const PathEntry rhs) const { return m_name.cmp(rhs.m_name); } + int opCmp(string rhs) const { return m_name.cmp(rhs); } +} + +private bool isValidFilename(string str) +{ + foreach( ch; str ) + if( ch == '/' || /*ch == ':' ||*/ ch == '\\' ) return false; + return true; +} + +/// Joins two path strings. subpath must be relative. +string joinPath(string basepath, string subpath) +{ + Path p1 = Path(basepath); + Path p2 = Path(subpath); + return (p1 ~ p2).toString(); +} + +/// Splits up a path string into its elements/folders +PathEntry[] splitPath(string path) +{ + if( path.startsWith("/") || path.startsWith("\\") ) path = path[1 .. $]; + if( path.empty ) return null; + if( path.endsWith("/") || path.endsWith("\\") ) path = path[0 .. $-1]; + + // count the number of path nodes + size_t nelements = 0; + foreach( i, char ch; path ) + if( ch == '\\' || ch == '/' ) + nelements++; + nelements++; + + // reserve space for the elements + auto elements = new PathEntry[nelements]; + + // read and return the elements + size_t startidx = 0; + size_t eidx = 0; + foreach( i, char ch; path ) + if( ch == '\\' || ch == '/' ){ + enforce(i - startidx > 0, "Empty path entries not allowed."); + elements[eidx++] = PathEntry(path[startidx .. i]); + startidx = i+1; + } + elements[eidx++] = PathEntry(path[startidx .. $]); + enforce(path.length - startidx > 0, "Empty path entries not allowed."); + assert(eidx == nelements); + return elements; +} diff --git a/source/dub/internal/vibecompat/inet/url.d b/source/dub/internal/vibecompat/inet/url.d new file mode 100644 index 0000000..a3739cf --- /dev/null +++ b/source/dub/internal/vibecompat/inet/url.d @@ -0,0 +1,277 @@ +/** + URL parsing routines. + + Copyright: © 2012 RejectedSoftware e.K. + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Sönke Ludwig +*/ +module dub.internal.vibecompat.inet.url; + +public import dub.internal.vibecompat.inet.path; + +import std.algorithm; +import std.array; +import std.conv; +import std.exception; +import std.string; +import std.uri; + + +/** + Represents a URL decomposed into its components. +*/ +struct Url { + private { + string m_schema; + string m_pathString; + Path m_path; + string m_host; + ushort m_port; + string m_username; + string m_password; + string m_queryString; + string m_anchor; + } + + /// Constructs a new URL object from its components. + this(string schema, string host, ushort port, Path path) + { + m_schema = schema; + m_host = host; + m_port = port; + m_path = path; + m_pathString = path.toString(); + } + /// ditto + this(string schema, Path path) + { + this(schema, null, 0, path); + } + + /** Constructs a URL from its string representation. + + TODO: additional validation required (e.g. valid host and user names and port) + */ + this(string url_string) + { + auto str = url_string; + enforce(str.length > 0, "Empty URL."); + if( str[0] != '/' ){ + auto idx = str.countUntil(':'); + enforce(idx > 0, "No schema in URL:"~str); + m_schema = str[0 .. idx]; + str = str[idx+1 .. $]; + bool requires_host = false; + + switch(m_schema){ + case "http": + case "https": + case "ftp": + case "spdy": + case "sftp": + case "file": + // proto://server/path style + enforce(str.startsWith("//"), "URL must start with proto://..."); + requires_host = true; + str = str[2 .. $]; + goto default; + default: + auto si = str.countUntil('/'); + if( si < 0 ) si = str.length; + auto ai = str[0 .. si].countUntil('@'); + sizediff_t hs = 0; + if( ai >= 0 ){ + hs = ai+1; + auto ci = str[0 .. ai].countUntil(':'); + if( ci >= 0 ){ + m_username = str[0 .. ci]; + m_password = str[ci+1 .. ai]; + } else m_username = str[0 .. ai]; + enforce(m_username.length > 0, "Empty user name in URL."); + } + + m_host = str[hs .. si]; + auto pi = m_host.countUntil(':'); + if(pi > 0) { + enforce(pi < m_host.length-1, "Empty port in URL."); + m_port = to!ushort(m_host[pi+1..$]); + m_host = m_host[0 .. pi]; + } + + enforce(!requires_host || m_schema == "file" || m_host.length > 0, + "Empty server name in URL."); + str = str[si .. $]; + } + } + + this.localURI = str; + } + /// ditto + static Url parse(string url_string) + { + return Url(url_string); + } + + /// The schema/protocol part of the URL + @property string schema() const { return m_schema; } + /// ditto + @property void schema(string v) { m_schema = v; } + + /// The path part of the URL in the original string form + @property string pathString() const { return m_pathString; } + + /// The path part of the URL + @property Path path() const { return m_path; } + /// ditto + @property void path(Path p) + { + m_path = p; + auto pstr = p.toString(); + m_pathString = pstr; + } + + /// The host part of the URL (depends on the schema) + @property string host() const { return m_host; } + /// ditto + @property void host(string v) { m_host = v; } + + /// The port part of the URL (optional) + @property ushort port() const { return m_port; } + /// ditto + @property port(ushort v) { m_port = v; } + + /// The user name part of the URL (optional) + @property string username() const { return m_username; } + /// ditto + @property void username(string v) { m_username = v; } + + /// The password part of the URL (optional) + @property string password() const { return m_password; } + /// ditto + @property void password(string v) { m_password = v; } + + /// The query string part of the URL (optional) + @property string queryString() const { return m_queryString; } + /// ditto + @property void queryString(string v) { m_queryString = v; } + + /// The anchor part of the URL (optional) + @property string anchor() const { return m_anchor; } + + /// The path part plus query string and anchor + @property string localURI() + const { + auto str = appender!string(); + str.reserve(m_pathString.length + 2 + queryString.length + anchor.length); + str.put(encode(path.toString())); + if( queryString.length ) { + str.put("?"); + str.put(queryString); + } + if( anchor.length ) { + str.put("#"); + str.put(anchor); + } + return str.data; + } + /// ditto + @property void localURI(string str) + { + auto ai = str.countUntil('#'); + if( ai >= 0 ){ + m_anchor = str[ai+1 .. $]; + str = str[0 .. ai]; + } + + auto qi = str.countUntil('?'); + if( qi >= 0 ){ + m_queryString = str[qi+1 .. $]; + str = str[0 .. qi]; + } + + m_pathString = str; + m_path = Path(decode(str)); + } + + /// The URL to the parent path with query string and anchor stripped. + @property Url parentUrl() const { + Url ret; + ret.schema = schema; + ret.host = host; + ret.port = port; + ret.username = username; + ret.password = password; + ret.path = path.parentPath; + return ret; + } + + /// Converts this URL object to its string representation. + string toString() + const { + import std.format; + auto dst = appender!string(); + dst.put(schema); + dst.put(":"); + switch(schema){ + default: break; + case "file": + case "http": + case "https": + case "ftp": + case "spdy": + case "sftp": + dst.put("//"); + break; + } + dst.put(host); + if( m_port > 0 ) formattedWrite(dst, ":%d", m_port); + dst.put(localURI); + return dst.data; + } + + bool startsWith(const Url rhs) const { + if( m_schema != rhs.m_schema ) return false; + if( m_host != rhs.m_host ) return false; + // FIXME: also consider user, port, querystring, anchor etc + return path.startsWith(rhs.m_path); + } + + Url opBinary(string OP)(Path rhs) const if( OP == "~" ) { return Url(m_schema, m_host, m_port, m_path ~ rhs); } + Url opBinary(string OP)(PathEntry rhs) const if( OP == "~" ) { return Url(m_schema, m_host, m_port, m_path ~ rhs); } + void opOpAssign(string OP)(Path rhs) if( OP == "~" ) { m_path ~= rhs; } + void opOpAssign(string OP)(PathEntry rhs) if( OP == "~" ) { m_path ~= rhs; } + + /// Tests two URLs for equality using '=='. + bool opEquals(ref const Url rhs) const { + if( m_schema != rhs.m_schema ) return false; + if( m_host != rhs.m_host ) return false; + if( m_path != rhs.m_path ) return false; + return true; + } + /// ditto + bool opEquals(const Url other) const { return opEquals(other); } + + int opCmp(ref const Url rhs) const { + if( m_schema != rhs.m_schema ) return m_schema.cmp(rhs.m_schema); + if( m_host != rhs.m_host ) return m_host.cmp(rhs.m_host); + if( m_path != rhs.m_path ) return m_path.opCmp(rhs.m_path); + return true; + } +} + +unittest { + auto url = Url.parse("https://www.example.net/index.html"); + assert(url.schema == "https", url.schema); + assert(url.host == "www.example.net", url.host); + assert(url.path == Path("/index.html"), url.path.toString()); + + url = Url.parse("http://jo.doe:password@sub.www.example.net:4711/sub2/index.html?query#anchor"); + assert(url.schema == "http", url.schema); + assert(url.username == "jo.doe", url.username); + assert(url.password == "password", url.password); + assert(url.port == 4711, to!string(url.port)); + assert(url.host == "sub.www.example.net", url.host); + assert(url.path.toString() == "/sub2/index.html", url.path.toString()); + assert(url.queryString == "query", url.queryString); + assert(url.anchor == "anchor", url.anchor); +} diff --git a/source/dub/internal/vibecompat/inet/urltransfer.d b/source/dub/internal/vibecompat/inet/urltransfer.d new file mode 100644 index 0000000..c5ced6f --- /dev/null +++ b/source/dub/internal/vibecompat/inet/urltransfer.d @@ -0,0 +1,37 @@ +/** + Downloading and uploading of data from/to URLs. + + Copyright: © 2012 RejectedSoftware e.K. + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Sönke Ludwig +*/ +module dub.internal.vibecompat.inet.urltransfer; + +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.inet.url; + +import std.exception; +import std.net.curl; +import std.string; + + +/** + Downloads a file from the specified URL. + + Any redirects will be followed until the actual file resource is reached or if the redirection + limit of 10 is reached. Note that only HTTP(S) is currently supported. +*/ +void download(string url, string filename) +{ + auto conn = HTTP(); + static if( is(typeof(&conn.verifyPeer)) ) + conn.verifyPeer = false; + std.net.curl.download(url, filename, conn); +} + +/// ditto +void download(Url url, Path filename) +{ + download(url.toString(), filename.toNativeString()); +} diff --git a/source/dub/package_.d b/source/dub/package_.d index 1c4105f..e69ce74 100644 --- a/source/dub/package_.d +++ b/source/dub/package_.d @@ -9,6 +9,10 @@ import dub.compilers.compiler; import dub.dependency; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.data.json; +import dub.internal.vibecompat.inet.url; import dub.utils; import std.algorithm; @@ -19,10 +23,6 @@ import std.range; import std.string; import std.traits : EnumMembers; -import vibecompat.core.log; -import vibecompat.core.file; -import vibecompat.data.json; -import vibecompat.inet.url; enum PackageJsonFilename = "package.json"; diff --git a/source/dub/packagemanager.d b/source/dub/packagemanager.d index 62cf0a0..0f78316 100644 --- a/source/dub/packagemanager.d +++ b/source/dub/packagemanager.d @@ -9,6 +9,10 @@ import dub.dependency; import dub.installation; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.data.json; +import dub.internal.vibecompat.inet.path; import dub.package_; import dub.utils; @@ -19,10 +23,6 @@ import std.file; import std.string; import std.zip; -import vibecompat.core.file; -import vibecompat.core.log; -import vibecompat.data.json; -import vibecompat.inet.path; enum JournalJsonFilename = "journal.json"; diff --git a/source/dub/packagesupplier.d b/source/dub/packagesupplier.d index 6cfdd9d..19559d2 100644 --- a/source/dub/packagesupplier.d +++ b/source/dub/packagesupplier.d @@ -7,20 +7,19 @@ */ module dub.packagesupplier; -import dub.utils; import dub.dependency; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.data.json; +import dub.internal.vibecompat.inet.url; +import dub.internal.vibecompat.inet.urltransfer; +import dub.utils; import std.file; import std.exception; import std.zip; import std.conv; -import vibecompat.core.log; -import vibecompat.core.file; -import vibecompat.data.json; -import vibecompat.inet.url; -import vibecompat.inet.urltransfer; - /// Supplies packages, this is done by supplying the latest possible version /// which is available. interface PackageSupplier { diff --git a/source/dub/project.d b/source/dub/project.d index d9e22be..83e4b43 100644 --- a/source/dub/project.d +++ b/source/dub/project.d @@ -10,6 +10,11 @@ import dub.compilers.compiler; import dub.dependency; import dub.installation; +import dub.internal.std.process; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.data.json; +import dub.internal.vibecompat.inet.url; import dub.utils; import dub.registry; import dub.package_; @@ -17,10 +22,6 @@ import dub.packagesupplier; import dub.generators.generator; -import vibecompat.core.file; -import vibecompat.core.log; -import vibecompat.data.json; -import vibecompat.inet.url; // todo: cleanup imports. import std.algorithm; @@ -32,7 +33,6 @@ import std.string; import std.typecons; import std.zip; -import stdx.process; /// Representing a full project, with a root Package and several dependencies. diff --git a/source/dub/registry.d b/source/dub/registry.d index 60381ac..2c80cb6 100644 --- a/source/dub/registry.d +++ b/source/dub/registry.d Binary files differ diff --git a/source/dub/utils.d b/source/dub/utils.d index 31932c3..0222f40 100644 --- a/source/dub/utils.d +++ b/source/dub/utils.d @@ -7,10 +7,11 @@ */ module dub.utils; -import vibecompat.core.file; -import vibecompat.core.log; -import vibecompat.data.json; -import vibecompat.inet.url; +import dub.internal.std.process; +import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.log; +import dub.internal.vibecompat.data.json; +import dub.internal.vibecompat.inet.url; // todo: cleanup imports. import std.array; @@ -20,7 +21,6 @@ import std.zip; import std.typecons; import std.conv; -import stdx.process; package bool isEmptyDir(Path p) { diff --git a/source/stdx/process.d b/source/stdx/process.d deleted file mode 100644 index 6c73f06..0000000 --- a/source/stdx/process.d +++ /dev/null @@ -1,2473 +0,0 @@ -// Written in the D programming language. - -/** -Functions for starting and interacting with other processes, and for -working with the current process' execution environment. - -Process_handling: -$(UL $(LI - $(LREF spawnProcess) spawns a new _process, optionally assigning it an - arbitrary set of standard input, output, and error streams. - The function returns immediately, leaving the child _process to execute - in parallel with its parent. All other functions in this module that - spawn processes are built around $(D spawnProcess).) -$(LI - $(LREF wait) makes the parent _process wait for a child _process to - terminate. In general one should always do this, to avoid - child _processes becoming "zombies" when the parent _process exits. - Scope guards are perfect for this – see the $(LREF spawnProcess) - documentation for examples.) -$(LI - $(LREF pipeProcess) also spawns a child _process which runs - in parallel with its parent. However, instead of taking - arbitrary streams, it automatically creates a set of - pipes that allow the parent to communicate with the child - through the child's standard input, output, and/or error streams. - This function corresponds roughly to C's $(D popen) function.) -$(LI - $(LREF execute) starts a new _process and waits for it - to complete before returning. Additionally, it captures - the _process' standard output and error streams and returns - the output of these as a string.) -$(LI - $(LREF spawnShell), $(LREF pipeShell) and $(LREF shell) work like - $(D spawnProcess), $(D pipeProcess) and $(D execute), respectively, - except that they take a single command string and run it through - the current user's default command interpreter. - $(D shell) corresponds roughly to C's $(D system) function.) -$(LI - $(LREF kill) attempts to terminate a running process.) -) -Unless the directory of the executable file is explicitly specified, all -functions will search for it in the directories specified in the PATH -environment variable. - -Other_functionality: -$(UL -$(LI - $(LREF pipe) is used to create unidirectional pipes.) -$(LI - $(LREF environment) is an interface through which the current process' - environment variables can be read and manipulated.) -) - -Authors: - $(LINK2 https://github.com/kyllingstad, Lars Tandle Kyllingstad), - $(LINK2 https://github.com/schveiguy, Steven Schveighoffer), - $(LINK2 https://github.com/cybershadow, Vladimir Panteleev) -Copyright: - Copyright (c) 2013, the authors. All rights reserved. -Source: - $(PHOBOSSRC std/_process.d) -Macros: - WIKI=Phobos/StdProcess - OBJECTREF=$(D $(LINK2 object.html#$0,$0)) -*/ -module stdx.process; - -version (Posix) -{ - import core.stdc.errno; - import core.stdc.string; - import core.sys.posix.stdio; - import core.sys.posix.unistd; - import core.sys.posix.sys.wait; -} -version (Windows) -{ - import core.stdc.stdio; - import core.sys.windows.windows; - import std.utf; - import std.windows.syserror; -} -import std.algorithm; -import std.array; -import std.conv; -import std.exception; -import std.path; -import std.stdio; -import std.string; -import std.typecons; - - -// When the DMC runtime is used, we have to use some custom functions -// to convert between Windows file handles and FILE*s. -version (Win32) version (DigitalMars) version = DMC_RUNTIME; - - -// Some of the following should be moved to druntime. -private: - -// Windows API declarations. -version (Windows) -{ - extern(Windows) BOOL GetHandleInformation(HANDLE hObject, - LPDWORD lpdwFlags); - extern(Windows) BOOL SetHandleInformation(HANDLE hObject, - DWORD dwMask, - DWORD dwFlags); - extern(Windows) BOOL TerminateProcess(HANDLE hProcess, - UINT uExitCode); - extern(Windows) LPWSTR* CommandLineToArgvW(LPCWSTR lpCmdLine, - int* pNumArgs); - enum - { - HANDLE_FLAG_INHERIT = 0x1, - HANDLE_FLAG_PROTECT_FROM_CLOSE = 0x2, - } - enum CREATE_UNICODE_ENVIRONMENT = 0x400; -} - -// Microsoft Visual C Runtime (MSVCRT) declarations. -version (Windows) -{ - version (DMC_RUNTIME) { } else - { - import core.stdc.stdint; - extern(C) - { - int _fileno(FILE* stream); - HANDLE _get_osfhandle(int fd); - int _open_osfhandle(HANDLE osfhandle, int flags); - FILE* _fdopen(int fd, const (char)* mode); - int _close(int fd); - } - enum - { - STDIN_FILENO = 0, - STDOUT_FILENO = 1, - STDERR_FILENO = 2, - } - enum - { - _O_RDONLY = 0x0000, - _O_APPEND = 0x0004, - _O_TEXT = 0x4000, - } - } -} - -// POSIX API declarations. -version (Posix) -{ - version (OSX) - { - // https://www.gnu.org/software/gnulib/manual/html_node/environ.html - extern(C) char*** _NSGetEnviron(); - __gshared const char** environ; - shared static this() { environ = *_NSGetEnviron(); } - } - else - { - // Made available by the C runtime: - extern(C) extern __gshared const char** environ; - } -} - - -// Actual module classes/functions start here. -public: - - -// ============================================================================= -// Functions and classes for process management. -// ============================================================================= - - -/** -Spawns a new _process, optionally assigning it an -arbitrary set of standard input, output, and error streams. -The function returns immediately, leaving the child _process to execute -in parallel with its parent. - -Command_line: -There are four overloads of this function. The first two take an array -of strings, $(D args), which should contain the program name as the -zeroth element and any command-line arguments in subsequent elements. -The third and fourth versions are included for convenience, and may be -used when there are no command-line arguments. They take a single string, -$(D program), which specifies the program name. - -Unless a directory is specified in $(D args[0]) or $(D program), -$(D spawnProcess) will search for the program in the directories listed -in the PATH environment variable. To run an executable in the current -directory, use $(D "./$(I executable_name)"). ---- -// Run an executable called "prog" located in the current working -// directory: -auto pid = spawnProcess("./prog"); -scope(exit) wait(pid); -// We can do something else while the program runs. The scope guard -// ensures that the process is waited for at the end of the scope. -... - -// Run DMD on the file "myprog.d", specifying a few compiler switches: -auto dmdPid = spawnProcess(["dmd", "-O", "-release", "-inline", "myprog.d" ]); -if (wait(dmdPid) != 0) - writeln("Compilation failed!"); ---- - -Environment_variables: -With the first and third $(D spawnProcess) overloads, one can specify -the environment variables of the child process using the $(D environmentVars) -parameter. With the second and fourth overload, the child process inherits -its parent's environment variables. - -To make the child inherit the parent's environment $(I plus) one or more -additional variables, first use $(D $(LREF environment).$(LREF toAA)) to -obtain an associative array that contains the parent's environment -variables, and add the new variables to it before passing it to -$(D spawnProcess). ---- -auto envVars = environment.toAA(); -envVars["FOO"] = "bar"; -wait(spawnProcess("prog", envVars)); ---- - -Standard_streams: -The optional arguments $(D stdin_), $(D stdout_) and $(D stderr_) may -be used to assign arbitrary $(XREF stdio,File) objects as the standard -input, output and error streams, respectively, of the child process. The -former must be opened for reading, while the latter two must be opened for -writing. The default is for the child process to inherit the standard -streams of its parent. ---- -// Run DMD on the file myprog.d, logging any error messages to a -// file named errors.log. -auto logFile = File("errors.log", "w"); -auto pid = spawnProcess(["dmd", "myprog.d"], - std.stdio.stdin, - std.stdio.stdout, - logFile); -if (wait(pid) != 0) - writeln("Compilation failed. See errors.log for details."); ---- - -Note that if you pass a $(D File) object that is $(I not) -one of the standard input/output/error streams of the parent process, -that stream will by default be $(I closed) in the parent process when -this function returns. See the $(LREF Config) documentation below for -information about how to disable this behaviour. - -Beware of buffering issues when passing $(D File) objects to -$(D spawnProcess). The child process will inherit the low-level raw -read/write offset associated with the underlying file descriptor, but -it will not be aware of any buffered data. In cases where this matters -(e.g. when a file should be aligned before being passed on to the -child process), it may be a good idea to use unbuffered streams, or at -least ensure all relevant buffers are flushed. - -Params: -args = An array which contains the program name as the first element - and any command-line arguments in the following elements. -program = The program name, $(I without) command-line arguments. -environmentVars = The environment variables for the child process may - be specified using this parameter. By default it is $(D null), - which means that, the child process inherits the environment of - the parent process. -stdin_ = The standard input stream of the child process. - This can be any $(XREF stdio,File) that is opened for reading. - By default the child process inherits the parent's input - stream. -stdout_ = The standard output stream of the child process. - This can be any $(XREF stdio,File) that is opened for writing. - By default the child process inherits the parent's output - stream. -stderr_ = The standard error stream of the child process. - This can be any $(XREF stdio,File) that is opened for writing. - By default the child process inherits the parent's error - stream. -config = Options that control the behaviour of $(D spawnProcess). - See the $(LREF Config) documentation for details. - -Returns: -A $(LREF Pid) object that corresponds to the spawned process. - -Throws: -$(LREF ProcessException) on failure to start the process.$(BR) -$(XREF stdio,StdioException) on failure to pass one of the streams - to the child process (Windows only).$(BR) -$(CXREF exception,RangeError) if $(D args) is empty. -*/ -Pid spawnProcess(in char[][] args, - const string[string] environmentVars, - File stdin_ = std.stdio.stdin, - File stdout_ = std.stdio.stdout, - File stderr_ = std.stdio.stderr, - Config config = Config.none) - @trusted // TODO: Should be @safe -{ - version (Windows) auto args2 = escapeShellArguments(args); - else version (Posix) alias args2 = args; - return spawnProcessImpl(args2, toEnvz(environmentVars), - stdin_, stdout_, stderr_, config); -} - -/// ditto -Pid spawnProcess(in char[][] args, - File stdin_ = std.stdio.stdin, - File stdout_ = std.stdio.stdout, - File stderr_ = std.stdio.stderr, - Config config = Config.none) - @trusted // TODO: Should be @safe -{ - version (Windows) auto args2 = escapeShellArguments(args); - else version (Posix) alias args2 = args; - return spawnProcessImpl(args2, null, stdin_, stdout_, stderr_, config); -} - -/// ditto -Pid spawnProcess(in char[] program, - const string[string] environmentVars, - File stdin_ = std.stdio.stdin, - File stdout_ = std.stdio.stdout, - File stderr_ = std.stdio.stderr, - Config config = Config.none) - @trusted -{ - return spawnProcess((&program)[0 .. 1], environmentVars, - stdin_, stdout_, stderr_, config); -} - -/// ditto -Pid spawnProcess(in char[] program, - File stdin_ = std.stdio.stdin, - File stdout_ = std.stdio.stdout, - File stderr_ = std.stdio.stderr, - Config config = Config.none) - @trusted -{ - return spawnProcess((&program)[0 .. 1], - stdin_, stdout_, stderr_, config); -} - -/* -Implementation of spawnProcess() for POSIX. - -envz should be a zero-terminated array of zero-terminated strings -on the form "var=value". -*/ -version (Posix) -private Pid spawnProcessImpl(in char[][] args, - const(char*)* envz, - File stdin_, - File stdout_, - File stderr_, - Config config) - @trusted // TODO: Should be @safe -{ - const(char)[] name = args[0]; - if (any!isDirSeparator(name)) - { - if (!isExecutable(name)) - throw new ProcessException(text("Not an executable file: ", name)); - } - else - { - name = searchPathFor(name); - if (name is null) - throw new ProcessException(text("Executable file not found: ", name)); - } - - // Convert program name and arguments to C-style strings. - auto argz = new const(char)*[args.length+1]; - argz[0] = toStringz(name); - foreach (i; 1 .. args.length) argz[i] = toStringz(args[i]); - argz[$-1] = null; - - // Use parent's environment variables? - if (envz is null) envz = environ; - - // Get the file descriptors of the streams. - // These could potentially be invalid, but that is OK. If so, later calls - // to dup2() and close() will just silently fail without causing any harm. - auto stdinFD = core.stdc.stdio.fileno(stdin_.getFP()); - auto stdoutFD = core.stdc.stdio.fileno(stdout_.getFP()); - auto stderrFD = core.stdc.stdio.fileno(stderr_.getFP()); - - auto id = fork(); - if (id < 0) - throw ProcessException.newFromErrno("Failed to spawn new process"); - if (id == 0) - { - // Child process - - // Redirect streams and close the old file descriptors. - // In the case that stderr is redirected to stdout, we need - // to backup the file descriptor since stdout may be redirected - // as well. - if (stderrFD == STDOUT_FILENO) stderrFD = dup(stderrFD); - dup2(stdinFD, STDIN_FILENO); - dup2(stdoutFD, STDOUT_FILENO); - dup2(stderrFD, STDERR_FILENO); - - // Close the old file descriptors, unless they are - // either of the standard streams. - if (stdinFD > STDERR_FILENO) close(stdinFD); - if (stdoutFD > STDERR_FILENO) close(stdoutFD); - if (stderrFD > STDERR_FILENO) close(stderrFD); - - // Execute program. - execve(argz[0], argz.ptr, envz); - - // If execution fails, exit as quickly as possible. - perror("spawnProcess(): Failed to execute program"); - _exit(1); - assert (0); - } - else - { - // Parent process: Close streams and return. - if (stdinFD > STDERR_FILENO && !(config & Config.noCloseStdin)) - stdin_.close(); - if (stdoutFD > STDERR_FILENO && !(config & Config.noCloseStdout)) - stdout_.close(); - if (stderrFD > STDERR_FILENO && !(config & Config.noCloseStderr)) - stderr_.close(); - return new Pid(id); - } -} - -/* -Implementation of spawnProcess() for Windows. - -commandLine must contain the entire command line, properly -quoted/escaped as required by CreateProcessW(). - -envz must be a pointer to a block of UTF-16 characters on the form -"var1=value1\0var2=value2\0...varN=valueN\0\0". -*/ -version (Windows) -private Pid spawnProcessImpl(in char[] commandLine, - LPVOID envz, - File stdin_, - File stdout_, - File stderr_, - Config config) - @trusted -{ - auto commandz = toUTFz!(wchar*)(commandLine); - - // Startup info for CreateProcessW(). - STARTUPINFO_W startinfo; - startinfo.cb = startinfo.sizeof; - startinfo.dwFlags = STARTF_USESTDHANDLES; - - // Extract file descriptors and HANDLEs from the streams and make the - // handles inheritable. - static void prepareStream(ref File file, DWORD stdHandle, string which, - out int fileDescriptor, out HANDLE handle) - { - fileDescriptor = _fileno(file.getFP()); - if (fileDescriptor < 0) handle = GetStdHandle(stdHandle); - else - { - version (DMC_RUNTIME) handle = _fdToHandle(fileDescriptor); - else /* MSVCRT */ handle = _get_osfhandle(fileDescriptor); - } - DWORD dwFlags; - GetHandleInformation(handle, &dwFlags); - if (!(dwFlags & HANDLE_FLAG_INHERIT)) - { - if (!SetHandleInformation(handle, - HANDLE_FLAG_INHERIT, - HANDLE_FLAG_INHERIT)) - { - throw new StdioException( - "Failed to make "~which~" stream inheritable by child process (" - ~sysErrorString(GetLastError()) ~ ')', - 0); - } - } - } - int stdinFD = -1, stdoutFD = -1, stderrFD = -1; - prepareStream(stdin_, STD_INPUT_HANDLE, "stdin" , stdinFD, startinfo.hStdInput ); - prepareStream(stdout_, STD_OUTPUT_HANDLE, "stdout", stdoutFD, startinfo.hStdOutput); - prepareStream(stderr_, STD_ERROR_HANDLE, "stderr", stderrFD, startinfo.hStdError ); - - // Create process. - PROCESS_INFORMATION pi; - DWORD dwCreationFlags = CREATE_UNICODE_ENVIRONMENT | - ((config & Config.gui) ? CREATE_NO_WINDOW : 0); - if (!CreateProcessW(null, commandz, null, null, true, dwCreationFlags, - envz, null, &startinfo, &pi)) - throw ProcessException.newFromLastError("Failed to spawn new process"); - - // figure out if we should close any of the streams - if (stdinFD > STDERR_FILENO && !(config & Config.noCloseStdin)) - stdin_.close(); - if (stdoutFD > STDERR_FILENO && !(config & Config.noCloseStdout)) - stdout_.close(); - if (stderrFD > STDERR_FILENO && !(config & Config.noCloseStderr)) - stderr_.close(); - - // close the thread handle in the process info structure - CloseHandle(pi.hThread); - - return new Pid(pi.dwProcessId, pi.hProcess); -} - -// Searches the PATH variable for the given executable file, -// (checking that it is in fact executable). -version (Posix) -private string searchPathFor(in char[] executable) - @trusted //TODO: @safe nothrow -{ - auto pathz = core.stdc.stdlib.getenv("PATH"); - if (pathz == null) return null; - - foreach (dir; splitter(to!string(pathz), ':')) - { - auto execPath = buildPath(dir, executable); - if (isExecutable(execPath)) return execPath; - } - - return null; -} - -// Converts a string[string] array to a C array of C strings -// on the form "key=value". -version (Posix) -private const(char)** toEnvz(const string[string] env) - @trusted //TODO: @safe pure nothrow -{ - alias const(char)* stringz_t; - auto envz = new stringz_t[](env.length+1); - int i = 0; - foreach (k, v; env) envz[i++] = (k~'='~v~'\0').ptr; - envz[i] = null; - return envz.ptr; -} - -// Converts a string[string] array to a block of 16-bit -// characters on the form "key=value\0key=value\0...\0\0" -version (Windows) -private LPVOID toEnvz(const string[string] env) - @trusted //TODO: @safe pure nothrow -{ - auto envz = appender!(wchar[])(); - foreach(k, v; env) - { - envz.put(k); - envz.put('='); - envz.put(v); - envz.put('\0'); - } - envz.put('\0'); - return envz.data.ptr; -} - -// Checks whether the file exists and can be executed by the -// current user. -version (Posix) -private bool isExecutable(in char[] path) @trusted //TODO: @safe nothrow -{ - return (access(toStringz(path), X_OK) == 0); -} - -unittest -{ - TestScript prog1 = "exit 0"; - assert (wait(spawnProcess(prog1.path)) == 0); - - TestScript prog2 = "exit 123"; - auto pid2 = spawnProcess([prog2.path]); - assert (wait(pid2) == 123); - assert (wait(pid2) == 123); // Exit code is cached. - - version (Windows) TestScript prog3 = - "if not -%1-==-foo- ( exit 1 ) - if not -%2-==-bar- ( exit 1 ) - exit 0"; - else version (Posix) TestScript prog3 = - `if test "$1" != "foo"; then exit 1; fi - if test "$2" != "bar"; then exit 1; fi - exit 0`; - assert (wait(spawnProcess([ prog3.path, "foo", "bar"])) == 0); - assert (wait(spawnProcess(prog3.path)) == 1); - - version (Windows) TestScript prog4 = - "if %hello%==world ( exit 0 ) - exit 1"; - version (Posix) TestScript prog4 = - "if test $hello = world; then exit 0; fi - exit 1"; - auto env = [ "hello" : "world" ]; - assert (wait(spawnProcess(prog4.path, env)) == 0); - assert (wait(spawnProcess([prog4.path], env)) == 0); - - version (Windows) TestScript prog5 = - "set /p INPUT= - echo %INPUT% output %1 - echo %INPUT% error %2 1>&2"; - else version (Posix) TestScript prog5 = - "read INPUT - echo $INPUT output $1 - echo $INPUT error $2 >&2"; - auto pipe5i = pipe(); - auto pipe5o = pipe(); - auto pipe5e = pipe(); - auto pid5 = spawnProcess([ prog5.path, "foo", "bar" ], - pipe5i.readEnd, pipe5o.writeEnd, pipe5e.writeEnd); - pipe5i.writeEnd.writeln("input"); - pipe5i.writeEnd.flush(); - assert (pipe5o.readEnd.readln().chomp() == "input output foo"); - assert (pipe5e.readEnd.readln().chomp().stripRight() == "input error bar"); - wait(pid5); - - import std.ascii, std.file, std.uuid; - auto path6i = buildPath(tempDir(), randomUUID().toString()); - auto path6o = buildPath(tempDir(), randomUUID().toString()); - auto path6e = buildPath(tempDir(), randomUUID().toString()); - std.file.write(path6i, "INPUT"~std.ascii.newline); - auto file6i = File(path6i, "r"); - auto file6o = File(path6o, "w"); - auto file6e = File(path6e, "w"); - auto pid6 = spawnProcess([prog5.path, "bar", "baz" ], - file6i, file6o, file6e); - wait(pid6); - assert (readText(path6o).chomp() == "INPUT output bar"); - assert (readText(path6e).chomp().stripRight() == "INPUT error baz"); - remove(path6i); - remove(path6o); - remove(path6e); -} - - -/** -A variation on $(LREF spawnProcess) that runs the given _command through -the current user's preferred _command interpreter (aka. shell). - -The string $(D command) is passed verbatim to the shell, and is therefore -subject to its rules about _command structure, argument/filename quoting -and escaping of special characters. -The path to the shell executable is determined by the $(LREF userShell) -function. - -In all other respects this function works just like $(D spawnProcess). -Please refer to the $(LREF spawnProcess) documentation for descriptions -of the other function parameters, the return value and any exceptions -that may be thrown. ---- -// Run the command/program "foo" on the file named "my file.txt", and -// redirect its output into foo.log. -auto pid = spawnShell(`foo "my file.txt" > foo.log`); -wait(pid); ---- - -See_also: -$(LREF escapeShellCommand), which may be helpful in constructing a -properly quoted and escaped shell command line for the current plattform, -from an array of separate arguments. -*/ -Pid spawnShell(in char[] command, - const string[string] environmentVars, - File stdin_ = std.stdio.stdin, - File stdout_ = std.stdio.stdout, - File stderr_ = std.stdio.stderr, - Config config = Config.none) - @trusted // TODO: Should be @safe -{ - return spawnShellImpl(command, toEnvz(environmentVars), - stdin_, stdout_, stderr_, config); -} - -/// ditto -Pid spawnShell(in char[] command, - File stdin_ = std.stdio.stdin, - File stdout_ = std.stdio.stdout, - File stderr_ = std.stdio.stderr, - Config config = Config.none) - @trusted // TODO: Should be @safe -{ - return spawnShellImpl(command, null, stdin_, stdout_, stderr_, config); -} - -// Implementation of spawnShell() for Windows. -version(Windows) -private Pid spawnShellImpl(in char[] command, - LPVOID envz, - File stdin_ = std.stdio.stdin, - File stdout_ = std.stdio.stdout, - File stderr_ = std.stdio.stderr, - Config config = Config.none) - @trusted // TODO: Should be @safe -{ - auto scmd = escapeShellArguments(userShell, shellSwitch) ~ " " ~ command; - return spawnProcessImpl(scmd, envz, stdin_, stdout_, stderr_, config); -} - -// Implementation of spawnShell() for POSIX. -version(Posix) -private Pid spawnShellImpl(in char[] command, - const char** envz, - File stdin_ = std.stdio.stdin, - File stdout_ = std.stdio.stdout, - File stderr_ = std.stdio.stderr, - Config config = Config.none) - @trusted // TODO: Should be @safe -{ - const(char)[][3] args; - args[0] = userShell; - args[1] = shellSwitch; - args[2] = command; - return spawnProcessImpl(args, envz, stdin_, stdout_, stderr_, config); -} - - - -/** -Flags that control the behaviour of $(LREF spawnProcess) and -$(LREF spawnShell). - -Use bitwise OR to combine flags. - -Example: ---- -auto logFile = File("myapp_error.log", "w"); - -// Start program in a console window (Windows only), redirect -// its error stream to logFile, and leave logFile open in the -// parent process as well. -auto pid = spawnProcess("myapp", stdin, stdout, logFile, - Config.noCloseStderr | Config.gui); -scope(exit) -{ - auto exitCode = wait(pid); - logFile.writeln("myapp exited with code ", exitCode); - logFile.close(); -} ---- -*/ -enum Config -{ - none = 0, - - /** - Unless the child process inherits the standard - input/output/error streams of its parent, one almost - always wants the streams closed in the parent when - $(LREF spawnProcess) returns. Therefore, by default, this - is done. If this is not desirable, pass any of these - options to spawnProcess. - */ - noCloseStdin = 1, - noCloseStdout = 2, /// ditto - noCloseStderr = 4, /// ditto - - /** - On Windows, the child process will by default be run in - a console window. This option wil cause it to run in "GUI mode" - instead, i.e., without a console. On POSIX, it has no effect. - */ - gui = 8, -} - - -/// A handle that corresponds to a spawned process. -final class Pid -{ - /** - The process ID number. - - This is a number that uniquely identifies the process on the operating - system, for at least as long as the process is running. Once $(LREF wait) - has been called on the $(LREF Pid), this method will return an - invalid process ID. - */ - @property int processID() const @safe pure nothrow - { - return _processID; - } - - /** - An operating system handle to the process. - - This handle is used to specify the process in OS-specific APIs. - On POSIX, this function returns a $(D core.sys.posix.sys.types.pid_t) - with the same value as $(LREF processID), while on Windows it returns - a $(D core.sys.windows.windows.HANDLE). - - Once $(LREF wait) has been called on the $(LREF Pid), this method - will return an invalid handle. - */ - // Note: Since HANDLE is a reference, this function cannot be const. - version (Windows) - @property HANDLE osHandle() @safe pure nothrow - { - return _handle; - } - else version (Posix) - @property pid_t osHandle() @safe pure nothrow - { - return _processID; - } - -private: - /* - Pid.performWait() does the dirty work for wait() and nonBlockingWait(). - - If block == true, this function blocks until the process terminates, - sets _processID to terminated, and returns the exit code or terminating - signal as described in the wait() documentation. - - If block == false, this function returns immediately, regardless - of the status of the process. If the process has terminated, the - function has the exact same effect as the blocking version. If not, - it returns 0 and does not modify _processID. - */ - version (Posix) - int performWait(bool block) @trusted - { - if (_processID == terminated) return _exitCode; - int exitCode; - while(true) - { - int status; - auto check = waitpid(_processID, &status, block ? 0 : WNOHANG); - if (check == -1) - { - if (errno == ECHILD) - { - throw new ProcessException( - "Process does not exist or is not a child process."); - } - else - { - // waitpid() was interrupted by a signal. We simply - // restart it. - assert (errno == EINTR); - continue; - } - } - if (!block && check == 0) return 0; - if (WIFEXITED(status)) - { - exitCode = WEXITSTATUS(status); - break; - } - else if (WIFSIGNALED(status)) - { - exitCode = -WTERMSIG(status); - break; - } - // We check again whether the call should be blocking, - // since we don't care about other status changes besides - // "exited" and "terminated by signal". - if (!block) return 0; - - // Process has stopped, but not terminated, so we continue waiting. - } - // Mark Pid as terminated, and cache and return exit code. - _processID = terminated; - _exitCode = exitCode; - return exitCode; - } - else version (Windows) - { - int performWait(bool block) @trusted - { - if (_processID == terminated) return _exitCode; - assert (_handle != INVALID_HANDLE_VALUE); - if (block) - { - auto result = WaitForSingleObject(_handle, INFINITE); - if (result != WAIT_OBJECT_0) - throw ProcessException.newFromLastError("Wait failed."); - } - if (!GetExitCodeProcess(_handle, cast(LPDWORD)&_exitCode)) - throw ProcessException.newFromLastError(); - if (!block && _exitCode == STILL_ACTIVE) return 0; - CloseHandle(_handle); - _handle = INVALID_HANDLE_VALUE; - _processID = terminated; - return _exitCode; - } - - ~this() - { - if(_handle != INVALID_HANDLE_VALUE) - { - CloseHandle(_handle); - _handle = INVALID_HANDLE_VALUE; - } - } - } - - // Special values for _processID. - enum invalid = -1, terminated = -2; - - // OS process ID number. Only nonnegative IDs correspond to - // running processes. - int _processID = invalid; - - // Exit code cached by wait(). This is only expected to hold a - // sensible value if _processID == terminated. - int _exitCode; - - // Pids are only meant to be constructed inside this module, so - // we make the constructor private. - version (Windows) - { - HANDLE _handle; - this(int pid, HANDLE handle) @safe pure nothrow - { - _processID = pid; - _handle = handle; - } - } - else - { - this(int id) @safe pure nothrow - { - _processID = id; - } - } -} - - -/** -Waits for the process associated with $(D pid) to terminate, and returns -its exit status. - -In general one should always _wait for child processes to terminate -before exiting the parent process. Otherwise, they may become -"$(WEB en.wikipedia.org/wiki/Zombie_process,zombies)" – processes -that are defunct, yet still occupy a slot in the OS process table. - -If the process has already terminated, this function returns directly. -The exit code is cached, so that if wait() is called multiple times on -the same $(LREF Pid) it will always return the same value. - -POSIX_specific: -If the process is terminated by a signal, this function returns a -negative number whose absolute value is the signal number. -Since POSIX restricts normal exit codes to the range 0-255, a -negative return value will always indicate termination by signal. -Signal codes are defined in the $(D core.sys.posix.signal) module -(which corresponds to the $(D signal.h) POSIX header). - -Throws: -$(LREF ProcessException) on failure. - -Examples: -See the $(LREF spawnProcess) documentation. - -See_also: -$(LREF tryWait), for a non-blocking function. -*/ -int wait(Pid pid) @safe -{ - assert(pid !is null, "Called wait on a null Pid."); - return pid.performWait(true); -} - - -/** -A non-blocking version of $(LREF wait). - -If the process associated with $(D pid) has already terminated, -$(D tryWait) has the exact same effect as $(D wait). -In this case, it returns a tuple where the $(D terminated) field -is set to $(D true) and the $(D status) field has the same -interpretation as the return value of $(D wait). - -If the process has $(I not) yet terminated, this function differs -from $(D wait) in that does not wait for this to happen, but instead -returns immediately. The $(D terminated) field of the returned -tuple will then be set to $(D false), while the $(D status) field -will always be 0 (zero). $(D wait) or $(D tryWait) should then be -called again on the same $(D Pid) at some later time; not only to -get the exit code, but also to avoid the process becoming a "zombie" -when it finally terminates. (See $(LREF wait) for details). - -Throws: -$(LREF ProcessException) on failure. - -Example: ---- -auto pid = spawnProcess("dmd myapp.d"); -scope(exit) wait(pid); -... -auto dmd = tryWait(pid); -if (dmd.terminated) -{ - if (dmd.status == 0) writeln("Compilation succeeded!"); - else writeln("Compilation failed"); -} -else writeln("Still compiling..."); -... ---- -Note that in this example, the first $(D wait) call will have no -effect if the process has already terminated by the time $(D tryWait) -is called. In the opposite case, however, the $(D scope) statement -ensures that we always wait for the process if it hasn't terminated -by the time we reach the end of the scope. -*/ -Tuple!(bool, "terminated", int, "status") tryWait(Pid pid) @safe -{ - assert(pid !is null, "Called tryWait on a null Pid."); - auto code = pid.performWait(false); - return typeof(return)(pid._processID == Pid.terminated, code); -} - - -/** -Attempts to terminate the process associated with $(D pid). - -The effect of this function, as well as the meaning of $(D codeOrSignal), -is highly platform dependent. Details are given below. Common to all -platforms is that this function only $(I initiates) termination of the process, -and returns immediately. It does not wait for the process to end, -nor does it guarantee that the process does in fact get terminated. - -Always call $(LREF wait) to wait for a process to complete, even if $(D kill) -has been called on it. - -Windows_specific: -The process will be -$(LINK2 http://msdn.microsoft.com/en-us/library/windows/desktop/ms686714%28v=vs.100%29.aspx, -forcefully and abruptly terminated). If $(D codeOrSignal) is specified, it -will be used as the exit code of the process. If not, the process wil exit -with code 1. Do not use $(D codeOrSignal = 259), as this is a special value -(aka. $(LINK2 http://msdn.microsoft.com/en-us/library/windows/desktop/ms683189.aspx, -STILL_ACTIVE)) used by Windows to signal that a process has in fact $(I not) -terminated yet. ---- -auto pid = spawnProcess("some_app"); -kill(pid, 10); -assert (wait(pid) == 10); ---- - -POSIX_specific: -A $(LINK2 http://en.wikipedia.org/wiki/Unix_signal,signal) will be sent to -the process, whose value is given by $(D codeOrSignal). Depending on the -signal sent, this may or may not terminate the process. Symbolic constants -for various $(LINK2 http://en.wikipedia.org/wiki/Unix_signal#POSIX_signals, -POSIX signals) are defined in $(D core.sys.posix.signal), which corresponds to the -$(LINK2 http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/signal.h.html, -$(D signal.h) POSIX header). If $(D codeOrSignal) is omitted, the -$(D SIGTERM) signal will be sent. (This matches the behaviour of the -$(LINK2 http://pubs.opengroup.org/onlinepubs/9699919799/utilities/kill.html, -$(D _kill)) shell command.) ---- -import core.sys.posix.signal: SIGKILL; -auto pid = spawnProcess("some_app"); -kill(pid, SIGKILL); -assert (wait(pid) == -SIGKILL); // Negative return value on POSIX! ---- - -Throws: -$(LREF ProcessException) if the operating system reports an error. - (Note that this does not include failure to terminate the process, - which is considered a "normal" outcome.)$(BR) -$(OBJECTREF Error) if $(D codeOrSignal) is negative. -*/ -void kill(Pid pid) -{ - version (Windows) kill(pid, 1); - else version (Posix) - { - import core.sys.posix.signal: SIGTERM; - kill(pid, SIGTERM); - } -} - -/// ditto -void kill(Pid pid, int codeOrSignal) -{ - version (Windows) enum errMsg = "Invalid exit code"; - else version (Posix) enum errMsg = "Invalid signal"; - if (codeOrSignal < 0) throw new Error(errMsg); - - version (Windows) - { - if (!TerminateProcess(pid.osHandle, codeOrSignal)) - throw ProcessException.newFromLastError(); - } - else version (Posix) - { - import core.sys.posix.signal; - if (kill(pid.osHandle, codeOrSignal) == -1) - throw ProcessException.newFromErrno(); - } -} - -unittest -{ - // The test script goes into an infinite loop. - version (Windows) - { - TestScript prog = "loop: - goto loop"; - } - else version (Posix) - { - import core.sys.posix.signal: SIGTERM, SIGKILL; - TestScript prog = "while true; do; done"; - } - auto pid = spawnProcess(prog.path); - kill(pid); - version (Windows) assert (wait(pid) == 1); - else version (Posix) assert (wait(pid) == -SIGTERM); - - pid = spawnProcess(prog.path); - auto s = tryWait(pid); - assert (!s.terminated && s.status == 0); - version (Windows) kill(pid, 123); - else version (Posix) kill(pid, SIGKILL); - do { s = tryWait(pid); } while (!s.terminated); - version (Windows) assert (s.status == 123); - else version (Posix) assert (s.status == -SIGKILL); -} - - -private File encapPipeAsFile(FILE* fil) -{ - static struct Impl - { - FILE * handle = null; // Is null iff this Impl is closed by another File - uint refs = uint.max / 2; - bool isPipe; - } - auto f = File.wrapFile(fil); - auto imp = *cast(Impl**)&f; - imp.refs = 1; - imp.isPipe = true; - return f; -} - -/** -Creates a unidirectional _pipe. - -Data is written to one end of the _pipe and read from the other. ---- -auto p = pipe(); -p.writeEnd.writeln("Hello World"); -assert (p.readEnd.readln().chomp() == "Hello World"); ---- -Pipes can, for example, be used for interprocess communication -by spawning a new process and passing one end of the _pipe to -the child, while the parent uses the other end. -(See also $(LREF pipeProcess) and $(LREF pipeShell) for an easier -way of doing this.) ---- -// Use cURL to download the dlang.org front page, pipe its -// output to grep to extract a list of links to ZIP files, -// and write the list to the file "D downloads.txt": -auto p = pipe(); -auto outFile = File("D downloads.txt", "w"); -auto cpid = spawnProcess(["curl", "http://dlang.org/download.html"], - std.stdio.stdin, p.writeEnd); -scope(exit) wait(cpid); -auto gpid = spawnProcess(["grep", "-o", `http://\S*\.zip`], - p.readEnd, outFile); -scope(exit) wait(gpid); ---- - -Returns: -A $(LREF Pipe) object that corresponds to the created _pipe. - -Throws: -$(XREF stdio,StdioException) on failure. -*/ -version (Posix) -Pipe pipe() @trusted //TODO: @safe -{ - int[2] fds; - errnoEnforce(core.sys.posix.unistd.pipe(fds) == 0, - "Unable to create pipe"); - Pipe p; - auto readFP = fdopen(fds[0], "r"); - if (readFP == null) - throw new StdioException("Cannot open read end of pipe"); - p._read = encapPipeAsFile(readFP); - auto writeFP = fdopen(fds[1], "w"); - if (writeFP == null) - throw new StdioException("Cannot open write end of pipe"); - p._write = encapPipeAsFile(writeFP); - return p; -} -else version (Windows) -Pipe pipe() @trusted //TODO: @safe -{ - // use CreatePipe to create an anonymous pipe - HANDLE readHandle; - HANDLE writeHandle; - if (!CreatePipe(&readHandle, &writeHandle, null, 0)) - { - throw new StdioException( - "Error creating pipe (" ~ sysErrorString(GetLastError()) ~ ')', - 0); - } - - // Create file descriptors from the handles - version (DMC_RUNTIME) - { - auto readFD = _handleToFD(readHandle, FHND_DEVICE); - auto writeFD = _handleToFD(writeHandle, FHND_DEVICE); - } - else // MSVCRT - { - auto readFD = _open_osfhandle(readHandle, _O_RDONLY); - auto writeFD = _open_osfhandle(writeHandle, _O_APPEND); - } - version (DMC_RUNTIME) alias .close _close; - if (readFD == -1 || writeFD == -1) - { - // Close file descriptors, then throw. - if (readFD >= 0) _close(readFD); - else CloseHandle(readHandle); - if (writeFD >= 0) _close(writeFD); - else CloseHandle(writeHandle); - throw new StdioException("Error creating pipe"); - } - - // Create FILE pointers from the file descriptors - Pipe p; - version (DMC_RUNTIME) - { - // This is a re-implementation of DMC's fdopen, but without the - // mucking with the file descriptor. POSIX standard requires the - // new fdopen'd file to retain the given file descriptor's - // position. - FILE * local_fdopen(int fd, const(char)* mode) - { - auto fp = core.stdc.stdio.fopen("NUL", mode); - if(!fp) return null; - FLOCK(fp); - auto iob = cast(_iobuf*)fp; - .close(iob._file); - iob._file = fd; - iob._flag &= ~_IOTRAN; - FUNLOCK(fp); - return fp; - } - - auto readFP = local_fdopen(readFD, "r"); - auto writeFP = local_fdopen(writeFD, "a"); - } - else // MSVCRT - { - auto readFP = _fdopen(readFD, "r"); - auto writeFP = _fdopen(writeFD, "a"); - } - if (readFP == null || writeFP == null) - { - // Close streams, then throw. - if (readFP != null) fclose(readFP); - else _close(readFD); - if (writeFP != null) fclose(writeFP); - else _close(writeFD); - throw new StdioException("Cannot open pipe"); - } - p._read = encapPipeAsFile(readFP); - p._write = encapPipeAsFile(writeFP); - return p; -} - - -/// An interface to a pipe created by the $(LREF pipe) function. -struct Pipe -{ - /// The read end of the pipe. - @property File readEnd() @trusted /*TODO: @safe nothrow*/ { return _read; } - - - /// The write end of the pipe. - @property File writeEnd() @trusted /*TODO: @safe nothrow*/ { return _write; } - - - /** - Closes both ends of the pipe. - - Normally it is not necessary to do this manually, as $(XREF stdio,File) - objects are automatically closed when there are no more references - to them. - - Note that if either end of the pipe has been passed to a child process, - it will only be closed in the parent process. (What happens in the - child process is platform dependent.) - */ - void close() @trusted //TODO: @safe nothrow - { - _read.close(); - _write.close(); - } - -private: - File _read, _write; -} - -unittest -{ - auto p = pipe(); - p.writeEnd.writeln("Hello World"); - p.writeEnd.flush(); - assert (p.readEnd.readln().chomp() == "Hello World"); -} - - -/** -Starts a new process, creating pipes to redirect its standard -input, output and/or error streams. - -These functions return immediately, leaving the child process to -execute in parallel with the parent. -$(LREF pipeShell) invokes the user's _command interpreter, as -determined by $(LREF userShell), to execute the given program or -_command. - -Returns: -A $(LREF ProcessPipes) object which contains $(XREF stdio,File) -handles that communicate with the redirected streams of the child -process, along with the $(LREF Pid) of the process. - -Throws: -$(LREF ProcessException) on failure to start the process.$(BR) -$(XREF stdio,StdioException) on failure to create pipes.$(BR) -$(OBJECTREF Error) if $(D redirectFlags) is an invalid combination of flags. - -Example: ---- -auto pipes = pipeProcess("my_application", Redirect.stdout | Redirect.stderr); -scope(exit) wait(pipes.pid); - -// Store lines of output. -string[] output; -foreach (line; pipes.stdout.byLine) output ~= line.idup; - -// Store lines of errors. -string[] errors; -foreach (line; pipes.stderr.byLine) errors ~= line.idup; ---- -*/ -ProcessPipes pipeProcess(string program, - Redirect redirectFlags = Redirect.all) - @trusted -{ - return pipeProcessImpl!spawnProcess(program, redirectFlags); -} - -/// ditto -ProcessPipes pipeProcess(string[] args, - Redirect redirectFlags = Redirect.all) - @trusted //TODO: @safe -{ - return pipeProcessImpl!spawnProcess(args, redirectFlags); -} - -/// ditto -ProcessPipes pipeShell(string command, Redirect redirectFlags = Redirect.all) - @safe -{ - return pipeProcessImpl!spawnShell(command, redirectFlags); -} - -// Implementation of the pipeProcess() family of functions. -private ProcessPipes pipeProcessImpl(alias spawnFunc, Cmd) - (Cmd command, Redirect redirectFlags) - @trusted //TODO: @safe -{ - File childStdin, childStdout, childStderr; - ProcessPipes pipes; - pipes._redirectFlags = redirectFlags; - - if (redirectFlags & Redirect.stdin) - { - auto p = pipe(); - childStdin = p.readEnd; - pipes._stdin = p.writeEnd; - } - else - { - childStdin = std.stdio.stdin; - } - - if (redirectFlags & Redirect.stdout) - { - if ((redirectFlags & Redirect.stdoutToStderr) != 0) - throw new Error("Invalid combination of options: Redirect.stdout | " - ~"Redirect.stdoutToStderr"); - auto p = pipe(); - childStdout = p.writeEnd; - pipes._stdout = p.readEnd; - } - else - { - childStdout = std.stdio.stdout; - } - - if (redirectFlags & Redirect.stderr) - { - if ((redirectFlags & Redirect.stderrToStdout) != 0) - throw new Error("Invalid combination of options: Redirect.stderr | " - ~"Redirect.stderrToStdout"); - auto p = pipe(); - childStderr = p.writeEnd; - pipes._stderr = p.readEnd; - } - else - { - childStderr = std.stdio.stderr; - } - - if (redirectFlags & Redirect.stdoutToStderr) - { - if (redirectFlags & Redirect.stderrToStdout) - { - // We know that neither of the other options have been - // set, so we assign the std.stdio.std* streams directly. - childStdout = std.stdio.stderr; - childStderr = std.stdio.stdout; - } - else - { - childStdout = childStderr; - } - } - else if (redirectFlags & Redirect.stderrToStdout) - { - childStderr = childStdout; - } - - pipes._pid = spawnFunc(command, null, childStdin, childStdout, childStderr); - return pipes; -} - - -/** -Flags that can be passed to $(LREF pipeProcess) and $(LREF pipeShell) -to specify which of the child process' standard streams are redirected. -Use bitwise OR to combine flags. -*/ -enum Redirect -{ - /// Redirect the standard input, output or error streams, respectively. - stdin = 1, - stdout = 2, /// ditto - stderr = 4, /// ditto - - /** - Redirect _all three streams. This is equivalent to - $(D Redirect.stdin | Redirect.stdout | Redirect.stderr). - */ - all = stdin | stdout | stderr, - - /** - Redirect the standard error stream into the standard output stream. - This can not be combined with $(D Redirect.stderr). - */ - stderrToStdout = 8, - - /** - Redirect the standard output stream into the standard error stream. - This can not be combined with $(D Redirect.stdout). - */ - stdoutToStderr = 16, -} - -unittest -{ - version (Windows) TestScript prog = - "call :sub %1 %2 0 - call :sub %1 %2 1 - call :sub %1 %2 2 - call :sub %1 %2 3 - exit 3 - - :sub - set /p INPUT= - if -%INPUT%-==-stop- ( exit %3 ) - echo %INPUT% %1 - echo %INPUT% %2 1>&2"; - else version (Posix) TestScript prog = - `for EXITCODE in 0 1 2 3; do - read INPUT - if test "$INPUT" = stop; then break; fi - echo "$INPUT $1" - echo "$INPUT $2" >&2 - done - exit $EXITCODE`; - auto pp = pipeProcess([prog.path, "bar", "baz"]); - pp.stdin.writeln("foo"); - pp.stdin.flush(); - assert (pp.stdout.readln().chomp() == "foo bar"); - assert (pp.stderr.readln().chomp().stripRight() == "foo baz"); - pp.stdin.writeln("1234567890"); - pp.stdin.flush(); - assert (pp.stdout.readln().chomp() == "1234567890 bar"); - assert (pp.stderr.readln().chomp().stripRight() == "1234567890 baz"); - pp.stdin.writeln("stop"); - pp.stdin.flush(); - assert (wait(pp.pid) == 2); - - pp = pipeProcess([prog.path, "12345", "67890"], - Redirect.stdin | Redirect.stdout | Redirect.stderrToStdout); - pp.stdin.writeln("xyz"); - pp.stdin.flush(); - assert (pp.stdout.readln().chomp() == "xyz 12345"); - assert (pp.stdout.readln().chomp().stripRight() == "xyz 67890"); - pp.stdin.writeln("stop"); - pp.stdin.flush(); - assert (wait(pp.pid) == 1); - - pp = pipeShell(prog.path~" AAAAA BBB", - Redirect.stdin | Redirect.stdoutToStderr | Redirect.stderr); - pp.stdin.writeln("ab"); - pp.stdin.flush(); - assert (pp.stderr.readln().chomp() == "ab AAAAA"); - assert (pp.stderr.readln().chomp().stripRight() == "ab BBB"); - pp.stdin.writeln("stop"); - pp.stdin.flush(); - assert (wait(pp.pid) == 1); -} - - -/** -Object which contains $(XREF stdio,File) handles that allow communication -with a child process through its standard streams. -*/ -struct ProcessPipes -{ - /// The $(LREF Pid) of the child process. - @property Pid pid() @safe nothrow - { - assert(_pid !is null); - return _pid; - } - - /** - An $(XREF stdio,File) that allows writing to the child process' - standard input stream. - - Throws: - $(OBJECTREF Error) if the child process' standard input stream hasn't - been redirected. - */ - @property File stdin() @trusted //TODO: @safe nothrow - { - if ((_redirectFlags & Redirect.stdin) == 0) - throw new Error("Child process' standard input stream hasn't " - ~"been redirected."); - return _stdin; - } - - /** - An $(XREF stdio,File) that allows reading from the child process' - standard output stream. - - Throws: - $(OBJECTREF Error) if the child process' standard output stream hasn't - been redirected. - */ - @property File stdout() @trusted //TODO: @safe nothrow - { - if ((_redirectFlags & Redirect.stdout) == 0) - throw new Error("Child process' standard output stream hasn't " - ~"been redirected."); - return _stdout; - } - - /** - An $(XREF stdio,File) that allows reading from the child process' - standard error stream. - - Throws: - $(OBJECTREF Error) if the child process' standard error stream hasn't - been redirected. - */ - @property File stderr() @trusted //TODO: @safe nothrow - { - if ((_redirectFlags & Redirect.stderr) == 0) - throw new Error("Child process' standard error stream hasn't " - ~"been redirected."); - return _stderr; - } - -private: - Redirect _redirectFlags; - Pid _pid; - File _stdin, _stdout, _stderr; -} - - -/** -Executes the given program and returns its exit code and output. - -This function blocks until the program terminates. -The $(D output) string includes what the program writes to its -standard error stream as well as its standard output stream. ---- -auto dmd = execute("dmd", "myapp.d"); -if (dmd.status != 0) writeln("Compilation failed:\n", dmd.output); ---- - -POSIX_specific: -If the process is terminated by a signal, the $(D status) field of -the return value will contain a negative number whose absolute -value is the signal number. (See $(LREF wait) for details.) - -Throws: -$(LREF ProcessException) on failure to start the process.$(BR) -$(XREF stdio,StdioException) on failure to capture output. -*/ -Tuple!(int, "status", string, "output") execute(string[] args...) - @trusted //TODO: @safe -{ - auto p = pipeProcess(args, Redirect.stdout | Redirect.stderrToStdout); - return processOutput(p, size_t.max); -} - -unittest -{ - // To avoid printing the newline characters, we use the echo|set trick on - // Windows, and printf on POSIX (neither echo -n nor echo \c are portable). - version (Windows) TestScript prog = - "echo|set /p=%1 - echo|set /p=%2 1>&2 - exit 123"; - else version (Posix) TestScript prog = - `printf '%s' $1 - printf '%s' $2 >&2 - exit 123`; - auto r = execute([prog.path, "foo", "bar"]); - assert (r.status == 123); - assert (r.output.stripRight() == "foobar"); - auto s = execute(prog.path, "Hello", "World"); - assert (s.status == 123); - assert (s.output.stripRight() == "HelloWorld"); -} - - -/** -Executes $(D _command) in the user's default _shell and returns its -exit code and output. - -This function blocks until the command terminates. -The $(D output) string includes what the command writes to its -standard error stream as well as its standard output stream. -The path to the _command interpreter is given by $(LREF userShell). ---- -auto ls = shell("ls -l"); -writefln("ls exited with code %s and said: %s", ls.status, ls.output); ---- - -POSIX_specific: -If the process is terminated by a signal, the $(D status) field of -the return value will contain a negative number whose absolute -value is the signal number. (See $(LREF wait) for details.) - -Throws: -$(LREF ProcessException) on failure to start the process.$(BR) -$(XREF stdio,StdioException) on failure to capture output. -*/ -Tuple!(int, "status", string, "output") shell(string command) - @trusted //TODO: @safe -{ - auto p = pipeShell(command, Redirect.stdout | Redirect.stderrToStdout); - return processOutput(p, size_t.max); -} - -unittest -{ - auto r1 = shell("echo foo"); - assert (r1.status == 0); - assert (r1.output.chomp() == "foo"); - auto r2 = shell("echo bar 1>&2"); - assert (r2.status == 0); - assert (r2.output.chomp().stripRight() == "bar"); - auto r3 = shell("exit 123"); - assert (r3.status == 123); - assert (r3.output.empty); -} - -// Collects the output and exit code for execute() and shell(). -private Tuple!(int, "status", string, "output") processOutput( - ref ProcessPipes pp, - size_t maxData) -{ - Appender!(ubyte[]) a; - enum chunkSize = 4096; - foreach (ubyte[] chunk; pp.stdout.byChunk(chunkSize)) - { - a.put(chunk); - if (a.data().length + chunkSize > maxData) break; - } - - typeof(return) r; - r.output = cast(string) a.data; - r.status = wait(pp.pid); - return r; -} - - - -/// An exception that signals a problem with starting or waiting for a process. -class ProcessException : Exception -{ - // Standard constructor. - this(string msg, string file = __FILE__, size_t line = __LINE__) - { - super(msg, file, line); - } - - // Creates a new ProcessException based on errno. - static ProcessException newFromErrno(string customMsg = null, - string file = __FILE__, - size_t line = __LINE__) - { - import core.stdc.errno; - import std.c.string; - version (linux) - { - char[1024] buf; - auto errnoMsg = to!string( - std.c.string.strerror_r(errno, buf.ptr, buf.length)); - } - else - { - auto errnoMsg = to!string(std.c.string.strerror(errno)); - } - auto msg = customMsg.empty() ? errnoMsg - : customMsg ~ " (" ~ errnoMsg ~ ')'; - return new ProcessException(msg, file, line); - } - - // Creates a new ProcessException based on GetLastError() (Windows only). - version (Windows) - static ProcessException newFromLastError(string customMsg = null, - string file = __FILE__, - size_t line = __LINE__) - { - auto lastMsg = sysErrorString(GetLastError()); - auto msg = customMsg.empty() ? lastMsg - : customMsg ~ " (" ~ lastMsg ~ ')'; - return new ProcessException(msg, file, line); - } -} - - -/** -Determines the path to the current user's default command interpreter. - -On Windows, this function returns the contents of the COMSPEC environment -variable, if it exists. Otherwise, it returns the string $(D "cmd.exe"). - -On POSIX, $(D userShell) returns the contents of the SHELL environment -variable, if it exists and is non-empty. Otherwise, it returns -$(D "/bin/sh"). -*/ -@property string userShell() @safe //TODO: nothrow -{ - version (Windows) return environment.get("COMSPEC", "cmd.exe"); - else version (Posix) return environment.get("SHELL", "/bin/sh"); -} - - -// A command-line switch that indicates to the shell that it should -// interpret the following argument as a command to be executed. -version (Posix) private immutable string shellSwitch = "-c"; -version (Windows) private immutable string shellSwitch = "/C"; - - -/// Returns the process ID number of the current process. -@property int thisProcessID() @trusted //TODO: @safe nothrow -{ - version (Windows) return GetCurrentProcessId(); - else version (Posix) return getpid(); -} - - -// Unittest support code: TestScript takes a string that contains a -// shell script for the current platform, and writes it to a temporary -// file. On Windows the file name gets a .cmd extension, while on -// POSIX its executable permission bit is set. The file is -// automatically deleted when the object goes out of scope. -version (unittest) -private struct TestScript -{ - this(string code) - { - import std.ascii, std.file, std.uuid; - version (Windows) - { - auto ext = ".cmd"; - auto firstLine = "@echo off"; - } - else version (Posix) - { - auto ext = ""; - auto firstLine = "#!/bin/sh"; - } - path = buildPath(tempDir(), randomUUID().toString()~ext); - std.file.write(path, firstLine~std.ascii.newline~code~std.ascii.newline); - version (Posix) - { - import core.sys.posix.sys.stat; - chmod(toStringz(path), octal!777); - } - } - - ~this() - { - import std.file; - if (!path.empty && exists(path)) remove(path); - } - - string path; -} - - -// ============================================================================= -// Functions for shell command quoting/escaping. -// ============================================================================= - - -/* - Command line arguments exist in three forms: - 1) string or char* array, as received by main. - Also used internally on POSIX systems. - 2) Command line string, as used in Windows' - CreateProcess and CommandLineToArgvW functions. - A specific quoting and escaping algorithm is used - to distinguish individual arguments. - 3) Shell command string, as written at a shell prompt - or passed to cmd /C - this one may contain shell - control characters, e.g. > or | for redirection / - piping - thus, yet another layer of escaping is - used to distinguish them from program arguments. - - Except for escapeWindowsArgument, the intermediary - format (2) is hidden away from the user in this module. -*/ - -/** -Escapes an argv-style argument array to be used with $(LREF spawnShell), -$(LREF pipeShell) or $(LREF shell). ---- -string url = "http://dlang.org/"; -shell(escapeShellCommand("wget", url, "-O", "dlang-index.html")); ---- - -Concatenate multiple $(D escapeShellCommand) and -$(LREF escapeShellFileName) results to use shell redirection or -piping operators. ---- -shell( - escapeShellCommand("curl", "http://dlang.org/download.html") ~ - "|" ~ - escapeShellCommand("grep", "-o", `http://\S*\.zip`) ~ - ">" ~ - escapeShellFileName("D download links.txt")); ---- -*/ -string escapeShellCommand(in char[][] args...) - //TODO: @safe pure nothrow -{ - return escapeShellCommandString(escapeShellArguments(args)); -} - - -private string escapeShellCommandString(string command) - //TODO: @safe pure nothrow -{ - version (Windows) - return escapeWindowsShellCommand(command); - else - return command; -} - -string escapeWindowsShellCommand(in char[] command) - //TODO: @safe pure nothrow (prevented by Appender) -{ - auto result = appender!string(); - result.reserve(command.length); - - foreach (c; command) - switch (c) - { - case '\0': - assert(0, "Cannot put NUL in command line"); - case '\r': - case '\n': - assert(0, "CR/LF are not escapable"); - case '\x01': .. case '\x09': - case '\x0B': .. case '\x0C': - case '\x0E': .. case '\x1F': - case '"': - case '^': - case '&': - case '<': - case '>': - case '|': - result.put('^'); - goto default; - default: - result.put(c); - } - return result.data; -} - -private string escapeShellArguments(in char[][] args...) - @trusted pure nothrow -{ - char[] buf; - - @safe nothrow - char[] allocator(size_t size) - { - if (buf.length == 0) - return buf = new char[size]; - else - { - auto p = buf.length; - buf.length = buf.length + 1 + size; - buf[p++] = ' '; - return buf[p..p+size]; - } - } - - foreach (arg; args) - escapeShellArgument!allocator(arg); - return assumeUnique(buf); -} - -private auto escapeShellArgument(alias allocator)(in char[] arg) @safe nothrow -{ - // The unittest for this function requires special - // preparation - see below. - - version (Windows) - return escapeWindowsArgumentImpl!allocator(arg); - else - return escapePosixArgumentImpl!allocator(arg); -} - -/** -Quotes a command-line argument in a manner conforming to the behavior of -$(LINK2 http://msdn.microsoft.com/en-us/library/windows/desktop/bb776391(v=vs.85).aspx, -CommandLineToArgvW). -*/ -string escapeWindowsArgument(in char[] arg) @trusted pure nothrow -{ - // Rationale for leaving this function as public: - // this algorithm of escaping paths is also used in other software, - // e.g. DMD's response files. - - auto buf = escapeWindowsArgumentImpl!charAllocator(arg); - return assumeUnique(buf); -} - - -private char[] charAllocator(size_t size) @safe pure nothrow -{ - return new char[size]; -} - - -private char[] escapeWindowsArgumentImpl(alias allocator)(in char[] arg) - @safe nothrow - if (is(typeof(allocator(size_t.init)[0] = char.init))) -{ - // References: - // * http://msdn.microsoft.com/en-us/library/windows/desktop/bb776391(v=vs.85).aspx - // * http://blogs.msdn.com/b/oldnewthing/archive/2010/09/17/10063629.aspx - - // Calculate the total string size. - - // Trailing backslashes must be escaped - bool escaping = true; - // Result size = input size + 2 for surrounding quotes + 1 for the - // backslash for each escaped character. - size_t size = 1 + arg.length + 1; - - foreach_reverse (c; arg) - { - if (c == '"') - { - escaping = true; - size++; - } - else - if (c == '\\') - { - if (escaping) - size++; - } - else - escaping = false; - } - - // Construct result string. - - auto buf = allocator(size); - size_t p = size; - buf[--p] = '"'; - escaping = true; - foreach_reverse (c; arg) - { - if (c == '"') - escaping = true; - else - if (c != '\\') - escaping = false; - - buf[--p] = c; - if (escaping) - buf[--p] = '\\'; - } - buf[--p] = '"'; - assert(p == 0); - - return buf; -} - -version(Windows) version(unittest) -{ - import core.sys.windows.windows; - import core.stdc.stddef; - - extern (Windows) wchar_t** CommandLineToArgvW(wchar_t*, int*); - extern (C) size_t wcslen(in wchar *); - - unittest - { - string[] testStrings = [ - `Hello`, - `Hello, world`, - `Hello, "world"`, - `C:\`, - `C:\dmd`, - `C:\Program Files\`, - ]; - - enum CHARS = `_x\" *&^`; // _ is placeholder for nothing - foreach (c1; CHARS) - foreach (c2; CHARS) - foreach (c3; CHARS) - foreach (c4; CHARS) - testStrings ~= [c1, c2, c3, c4].replace("_", ""); - - foreach (s; testStrings) - { - auto q = escapeWindowsArgument(s); - LPWSTR lpCommandLine = (to!(wchar[])("Dummy.exe " ~ q) ~ "\0"w).ptr; - int numArgs; - LPWSTR* args = CommandLineToArgvW(lpCommandLine, &numArgs); - scope(exit) LocalFree(args); - assert(numArgs==2, s ~ " => " ~ q ~ " #" ~ text(numArgs-1)); - auto arg = to!string(args[1][0..wcslen(args[1])]); - assert(arg == s, s ~ " => " ~ q ~ " => " ~ arg); - } - } -} - -private string escapePosixArgument(in char[] arg) @trusted pure nothrow -{ - auto buf = escapePosixArgumentImpl!charAllocator(arg); - return assumeUnique(buf); -} - -private char[] escapePosixArgumentImpl(alias allocator)(in char[] arg) - @safe nothrow - if (is(typeof(allocator(size_t.init)[0] = char.init))) -{ - // '\'' means: close quoted part of argument, append an escaped - // single quote, and reopen quotes - - // Below code is equivalent to: - // return `'` ~ std.array.replace(arg, `'`, `'\''`) ~ `'`; - - size_t size = 1 + arg.length + 1; - foreach (c; arg) - if (c == '\'') - size += 3; - - auto buf = allocator(size); - size_t p = 0; - buf[p++] = '\''; - foreach (c; arg) - if (c == '\'') - { - buf[p..p+4] = `'\''`; - p += 4; - } - else - buf[p++] = c; - buf[p++] = '\''; - assert(p == size); - - return buf; -} - -/** -Escapes a filename to be used for shell redirection with $(LREF spawnShell), -$(LREF pipeShell) or $(LREF shell). -*/ -string escapeShellFileName(in char[] fileName) @trusted pure nothrow -{ - // The unittest for this function requires special - // preparation - see below. - - version (Windows) - return cast(string)('"' ~ fileName ~ '"'); - else - return escapePosixArgument(fileName); -} - -// Loop generating strings with random characters -//version = unittest_burnin; - -version(unittest_burnin) -unittest -{ - // There are no readily-available commands on all platforms suitable - // for properly testing command escaping. The behavior of CMD's "echo" - // built-in differs from the POSIX program, and Windows ports of POSIX - // environments (Cygwin, msys, gnuwin32) may interfere with their own - // "echo" ports. - - // To run this unit test, create std_process_unittest_helper.d with the - // following content and compile it: - // import std.stdio, std.array; void main(string[] args) { write(args.join("\0")); } - // Then, test this module with: - // rdmd --main -unittest -version=unittest_burnin process.d - - auto helper = absolutePath("std_process_unittest_helper"); - assert(shell(helper ~ " hello").split("\0")[1..$] == ["hello"], "Helper malfunction"); - - void test(string[] s, string fn) - { - string e; - string[] g; - - e = escapeShellCommand(helper ~ s); - { - scope(failure) writefln("shell() failed.\nExpected:\t%s\nEncoded:\t%s", s, [e]); - g = shell(e).split("\0")[1..$]; - } - assert(s == g, format("shell() test failed.\nExpected:\t%s\nGot:\t\t%s\nEncoded:\t%s", s, g, [e])); - - e = escapeShellCommand(helper ~ s) ~ ">" ~ escapeShellFileName(fn); - { - scope(failure) writefln("system() failed.\nExpected:\t%s\nFilename:\t%s\nEncoded:\t%s", s, [fn], [e]); - system(e); - g = readText(fn).split("\0")[1..$]; - } - remove(fn); - assert(s == g, format("system() test failed.\nExpected:\t%s\nGot:\t\t%s\nEncoded:\t%s", s, g, [e])); - } - - while (true) - { - string[] args; - foreach (n; 0..uniform(1, 4)) - { - string arg; - foreach (l; 0..uniform(0, 10)) - { - dchar c; - while (true) - { - version (Windows) - { - // As long as DMD's system() uses CreateProcessA, - // we can't reliably pass Unicode - c = uniform(0, 128); - } - else - c = uniform!ubyte(); - - if (c == 0) - continue; // argv-strings are zero-terminated - version (Windows) - if (c == '\r' || c == '\n') - continue; // newlines are unescapable on Windows - break; - } - arg ~= c; - } - args ~= arg; - } - - // generate filename - string fn = "test_"; - foreach (l; 0..uniform(1, 10)) - { - dchar c; - while (true) - { - version (Windows) - c = uniform(0, 128); // as above - else - c = uniform!ubyte(); - - if (c == 0 || c == '/') - continue; // NUL and / are the only characters - // forbidden in POSIX filenames - version (Windows) - if (c < '\x20' || c == '<' || c == '>' || c == ':' || - c == '"' || c == '\\' || c == '|' || c == '?' || c == '*') - continue; // http://msdn.microsoft.com/en-us/library/aa365247(VS.85).aspx - break; - } - - fn ~= c; - } - - test(args, fn); - } -} - - -// ============================================================================= -// Environment variable manipulation. -// ============================================================================= - - -/** -Manipulates _environment variables using an associative-array-like -interface. - -This class contains only static methods, and cannot be instantiated. -See below for examples of use. -*/ -abstract final class environment -{ -static: - /** - Retrieves the value of the environment variable with the given $(D name). - - If no such variable exists, this function throws an $(D Exception). - See also $(LREF get), which doesn't throw on failure. - --- - auto path = environment["PATH"]; - --- - */ - string opIndex(string name) @safe - { - string value; - enforce(getImpl(name, value), "Environment variable not found: "~name); - return value; - } - - /** - Retrieves the value of the environment variable with the given $(D name), - or a default value if the variable doesn't exist. - - Unlike $(LREF opIndex), this function never throws. - --- - auto sh = environment.get("SHELL", "/bin/sh"); - --- - This function is also useful in checking for the existence of an - environment variable. - --- - auto myVar = environment.get("MYVAR"); - if (myVar is null) - { - // Environment variable doesn't exist. - // Note that we have to use 'is' for the comparison, since - // myVar == null is also true if the variable exists but is - // empty. - } - --- - */ - string get(string name, string defaultValue = null) @safe //TODO: nothrow - { - string value; - auto found = getImpl(name, value); - return found ? value : defaultValue; - } - - /** - Assigns the given $(D value) to the environment variable with the given - $(D name). - - If the variable does not exist, it will be created. If it already exists, - it will be overwritten. - --- - environment["foo"] = "bar"; - --- - */ - string opIndexAssign(string value, string name) @trusted - { - version (Posix) - { - if (core.sys.posix.stdlib.setenv(toStringz(name), toStringz(value), 1) != -1) - { - return value; - } - // The default errno error message is very uninformative - // in the most common case, so we handle it manually. - enforce(errno != EINVAL, - "Invalid environment variable name: '"~name~"'"); - errnoEnforce(false, - "Failed to add environment variable"); - assert(0); - } - else version (Windows) - { - enforce( - SetEnvironmentVariableW(toUTF16z(name), toUTF16z(value)), - sysErrorString(GetLastError()) - ); - return value; - } - else static assert(0); - } - - /** - Removes the environment variable with the given $(D name). - - If the variable isn't in the environment, this function returns - successfully without doing anything. - */ - void remove(string name) @trusted // TODO: @safe nothrow - { - version (Windows) SetEnvironmentVariableW(toUTF16z(name), null); - else version (Posix) core.sys.posix.stdlib.unsetenv(toStringz(name)); - else static assert(0); - } - - /** - Copies all environment variables into an associative array. - - Windows_specific: - While Windows environment variable names are case insensitive, D's - built-in associative arrays are not. This function will store all - variable names in uppercase (e.g. $(D PATH)). - */ - string[string] toAA() @trusted - { - string[string] aa; - version (Posix) - { - for (int i=0; environ[i] != null; ++i) - { - immutable varDef = to!string(environ[i]); - immutable eq = std.string.indexOf(varDef, '='); - assert (eq >= 0); - - immutable name = varDef[0 .. eq]; - immutable value = varDef[eq+1 .. $]; - - // In POSIX, environment variables may be defined more - // than once. This is a security issue, which we avoid - // by checking whether the key already exists in the array. - // For more info: - // http://www.dwheeler.com/secure-programs/Secure-Programs-HOWTO/environment-variables.html - if (name !in aa) aa[name] = value; - } - } - else version (Windows) - { - auto envBlock = GetEnvironmentStringsW(); - enforce(envBlock, "Failed to retrieve environment variables."); - scope(exit) FreeEnvironmentStringsW(envBlock); - - for (int i=0; envBlock[i] != '\0'; ++i) - { - auto start = i; - while (envBlock[i] != '=') ++i; - immutable name = toUTF8(toUpper(envBlock[start .. i])); - - start = i+1; - while (envBlock[i] != '\0') ++i; - // Just like in POSIX systems, environment variables may be - // defined more than once in an environment block on Windows, - // and it is just as much of a security issue there. Moreso, - // in fact, due to the case insensensitivity of variable names, - // which is not handled correctly by all programs. - if (name !in aa) aa[name] = toUTF8(envBlock[start .. i]); - } - } - else static assert(0); - return aa; - } - -private: - // Returns the length of an environment variable (in number of - // wchars, including the null terminator), or 0 if it doesn't exist. - version (Windows) - int varLength(LPCWSTR namez) @trusted nothrow - { - return GetEnvironmentVariableW(namez, null, 0); - } - - // Retrieves the environment variable, returns false on failure. - bool getImpl(string name, out string value) @trusted //TODO: nothrow - { - version (Windows) - { - const namez = toUTF16z(name); - immutable len = varLength(namez); - if (len == 0) return false; - if (len == 1) - { - value = ""; - return true; - } - - auto buf = new WCHAR[len]; - GetEnvironmentVariableW(namez, buf.ptr, to!DWORD(buf.length)); - value = toUTF8(buf[0 .. $-1]); - return true; - } - else version (Posix) - { - const vz = core.sys.posix.stdlib.getenv(toStringz(name)); - if (vz == null) return false; - auto v = vz[0 .. strlen(vz)]; - - // Cache the last call's result. - static string lastResult; - if (v != lastResult) lastResult = v.idup; - value = lastResult; - return true; - } - else static assert(0); - } -} - -unittest -{ - // New variable - environment["std_process"] = "foo"; - assert (environment["std_process"] == "foo"); - - // Set variable again - environment["std_process"] = "bar"; - assert (environment["std_process"] == "bar"); - - // Remove variable - environment.remove("std_process"); - - // Remove again, should succeed - environment.remove("std_process"); - - // Throw on not found. - assertThrown(environment["std_process"]); - - // get() without default value - assert (environment.get("std_process") == null); - - // get() with default value - assert (environment.get("std_process", "baz") == "baz"); - - // Convert to associative array - auto aa = environment.toAA(); - assert (aa.length > 0); - foreach (n, v; aa) - { - // Wine has some bugs related to environment variables: - // - Wine allows the existence of an env. variable with the name - // "\0", but GetEnvironmentVariable refuses to retrieve it. - // - If an env. variable has zero length, i.e. is "\0", - // GetEnvironmentVariable should return 1. Instead it returns - // 0, indicating the variable doesn't exist. - version (Windows) if (n.length == 0 || v.length == 0) continue; - - assert (v == environment[n]); - } -} diff --git a/source/vibecompat/core/file.d b/source/vibecompat/core/file.d deleted file mode 100644 index 6d3da32..0000000 --- a/source/vibecompat/core/file.d +++ /dev/null @@ -1,291 +0,0 @@ -/** - File handling. - - Copyright: © 2012 RejectedSoftware e.K. - License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. - Authors: Sönke Ludwig -*/ -module vibecompat.core.file; - -public import vibecompat.inet.url; -public import std.stdio; - -import vibecompat.core.log; - -import std.conv; -import std.c.stdio; -import std.datetime; -import std.exception; -import std.file; -import std.path; -import std.string; -import std.utf; - - -version(Posix){ - private extern(C) int mkstemps(char* templ, int suffixlen); -} - - -/* Add output range support to File -*/ -struct RangeFile { - File file; - alias file this; - - void put(in ubyte[] bytes) { file.rawWrite(bytes); } - void put(in char[] str) { put(cast(ubyte[])str); } - void put(char ch) { put((&ch)[0 .. 1]); } - void put(dchar ch) { char[4] chars; put(chars[0 .. encode(chars, ch)]); } - - ubyte[] readAll() - { - file.seek(0, SEEK_END); - auto sz = file.tell(); - enforce(sz <= size_t.max, "File is too big to read to memory."); - file.seek(0, SEEK_SET); - auto ret = new ubyte[cast(size_t)sz]; - return file.rawRead(ret); - } -} - - -/** - Opens a file stream with the specified mode. -*/ -RangeFile openFile(Path path, FileMode mode = FileMode.Read) -{ - string strmode; - final switch(mode){ - case FileMode.Read: strmode = "rb"; break; - case FileMode.ReadWrite: strmode = "rb+"; break; - case FileMode.CreateTrunc: strmode = "wb+"; break; - case FileMode.Append: strmode = "ab"; break; - } - auto ret = File(path.toNativeString(), strmode); - assert(ret.isOpen()); - return RangeFile(ret); -} -/// ditto -RangeFile openFile(string path, FileMode mode = FileMode.Read) -{ - return openFile(Path(path), mode); -} - -/** - Creates and opens a temporary file for writing. -*/ -RangeFile createTempFile(string suffix = null) -{ - version(Windows){ - char[L_tmpnam] tmp; - tmpnam(tmp.ptr); - auto tmpname = to!string(tmp.ptr); - if( tmpname.startsWith("\\") ) tmpname = tmpname[1 .. $]; - tmpname ~= suffix; - logDebug("tmp %s", tmpname); - return openFile(tmpname, FileMode.CreateTrunc); - } else { - import core.sys.posix.stdio; - enum pattern ="/tmp/vtmp.XXXXXX"; - scope templ = new char[pattern.length+suffix.length+1]; - templ[0 .. pattern.length] = pattern; - templ[pattern.length .. $-1] = suffix; - templ[$-1] = '\0'; - assert(suffix.length <= int.max); - auto fd = mkstemps(templ.ptr, cast(int)suffix.length); - enforce(fd >= 0, "Failed to create temporary file."); - auto ret = File.wrapFile(fdopen(fd, "wb+")); - return RangeFile(ret); - } -} - -/** - Moves or renames a file. -*/ -void moveFile(Path from, Path to) -{ - moveFile(from.toNativeString(), to.toNativeString()); -} -/// ditto -void moveFile(string from, string to) -{ - std.file.rename(from, to); -} - -/** - Copies a file. - - Note that attributes and time stamps are currently not retained. - - Params: - from = Path of the source file - to = Path for the destination file - overwrite = If true, any file existing at the destination path will be - overwritten. If this is false, an excpetion will be thrown should - a file already exist at the destination path. - - Throws: - An Exception if the copy operation fails for some reason. -*/ -void copyFile(Path from, Path to, bool overwrite = false) -{ - enforce(overwrite || !existsFile(to), "Destination file already exists."); - .copy(from.toNativeString(), to.toNativeString()); -} -/// ditto -void copyFile(string from, string to) -{ - copyFile(Path(from), Path(to)); -} - -/** - Removes a file -*/ -void removeFile(Path path) -{ - removeFile(path.toNativeString()); -} -/// ditto -void removeFile(string path) { - std.file.remove(path); -} - -/** - Checks if a file exists -*/ -bool existsFile(Path path) { - return existsFile(path.toNativeString()); -} -/// ditto -bool existsFile(string path) -{ - return std.file.exists(path); -} - -/** Stores information about the specified file/directory into 'info' - - Returns false if the file does not exist. -*/ -FileInfo getFileInfo(Path path) -{ - auto ent = std.file.dirEntry(path.toNativeString()); - return makeFileInfo(ent); -} -/// ditto -FileInfo getFileInfo(string path) -{ - return getFileInfo(Path(path)); -} - -/** - Creates a new directory. -*/ -void createDirectory(Path path) -{ - mkdir(path.toNativeString()); -} -/// ditto -void createDirectory(string path) -{ - createDirectory(Path(path)); -} - -/** - Enumerates all files in the specified directory. -*/ -void listDirectory(Path path, scope bool delegate(FileInfo info) del) -{ - foreach( DirEntry ent; dirEntries(path.toNativeString(), SpanMode.shallow) ) - if( !del(makeFileInfo(ent)) ) - break; -} -/// ditto -void listDirectory(string path, scope bool delegate(FileInfo info) del) -{ - listDirectory(Path(path), del); -} -/// ditto -int delegate(scope int delegate(ref FileInfo)) iterateDirectory(Path path) -{ - int iterator(scope int delegate(ref FileInfo) del){ - int ret = 0; - listDirectory(path, (fi){ - ret = del(fi); - return ret == 0; - }); - return ret; - } - return &iterator; -} -/// ditto -int delegate(scope int delegate(ref FileInfo)) iterateDirectory(string path) -{ - return iterateDirectory(Path(path)); -} - - -/** - Returns the current working directory. -*/ -Path getWorkingDirectory() -{ - return Path(std.file.getcwd()); -} - - -/** Contains general information about a file. -*/ -struct FileInfo { - /// Name of the file (not including the path) - string name; - - /// Size of the file (zero for directories) - ulong size; - - /// Time of the last modification - SysTime timeModified; - - /// Time of creation (not available on all operating systems/file systems) - SysTime timeCreated; - - /// True if this is a symlink to an actual file - bool isSymlink; - - /// True if this is a directory or a symlink pointing to a directory - bool isDirectory; -} - -/** - Specifies how a file is manipulated on disk. -*/ -enum FileMode { - /// The file is opened read-only. - Read, - /// The file is opened for read-write random access. - ReadWrite, - /// The file is truncated if it exists and created otherwise and the opened for read-write access. - CreateTrunc, - /// The file is opened for appending data to it and created if it does not exist. - Append -} - -/** - Accesses the contents of a file as a stream. -*/ - -private FileInfo makeFileInfo(DirEntry ent) -{ - FileInfo ret; - ret.name = baseName(ent.name); - if( ret.name.length == 0 ) ret.name = ent.name; - assert(ret.name.length > 0); - ret.size = ent.size; - ret.timeModified = ent.timeLastModified; - version(Windows) ret.timeCreated = ent.timeCreated; - else ret.timeCreated = ent.timeLastModified; - ret.isSymlink = ent.isSymlink; - ret.isDirectory = ent.isDir; - return ret; -} - diff --git a/source/vibecompat/core/log.d b/source/vibecompat/core/log.d deleted file mode 100644 index c015bae..0000000 --- a/source/vibecompat/core/log.d +++ /dev/null @@ -1,97 +0,0 @@ -/** - Central logging facility for vibe. - - Copyright: © 2012 RejectedSoftware e.K. - License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. - Authors: Sönke Ludwig -*/ -module vibecompat.core.log; - -import std.array; -import std.datetime; -import std.format; -import std.stdio; -import core.thread; - -private { - shared LogLevel s_minLevel = LogLevel.Info; - shared LogLevel s_logFileLevel; - shared bool s_plainLogging = false; -} - -/// Sets the minimum log level to be printed. -void setLogLevel(LogLevel level) nothrow -{ - s_minLevel = level; -} - -/// Disables output of thread/task ids with each log message -void setPlainLogging(bool enable) -{ - s_plainLogging = enable; -} - -/** - Logs a message. - - Params: - level = The log level for the logged message - fmt = See http://dlang.org/phobos/std_format.html#format-string -*/ -void logTrace(T...)(string fmt, auto ref T args) nothrow { log(LogLevel.Trace, fmt, args); } -/// ditto -void logDebug(T...)(string fmt, auto ref T args) nothrow { log(LogLevel.Debug, fmt, args); } -/// ditto -void logInfo(T...)(string fmt, auto ref T args) nothrow { log(LogLevel.Info, fmt, args); } -/// ditto -void logWarn(T...)(string fmt, auto ref T args) nothrow { log(LogLevel.Warn, fmt, args); } -/// ditto -void logError(T...)(string fmt, auto ref T args) nothrow { log(LogLevel.Error, fmt, args); } - -/// ditto -void log(T...)(LogLevel level, string fmt, auto ref T args) -nothrow { - if( level < s_minLevel ) return; - string pref; - final switch( level ){ - case LogLevel.Trace: pref = "trc"; break; - case LogLevel.Debug: pref = "dbg"; break; - case LogLevel.Info: pref = "INF"; break; - case LogLevel.Warn: pref = "WRN"; break; - case LogLevel.Error: pref = "ERR"; break; - case LogLevel.Fatal: pref = "FATAL"; break; - case LogLevel.None: assert(false); - } - - try { - auto txt = appender!string(); - txt.reserve(256); - formattedWrite(txt, fmt, args); - - auto threadid = cast(ulong)cast(void*)Thread.getThis(); - auto fiberid = cast(ulong)cast(void*)Fiber.getThis(); - threadid ^= threadid >> 32; - fiberid ^= fiberid >> 32; - - if( level >= s_minLevel ){ - if( s_plainLogging ) writeln(txt.data()); - else writefln("[%08X:%08X %s] %s", threadid, fiberid, pref, txt.data()); - stdout.flush(); - } - } catch( Exception e ){ - // this is bad but what can we do.. - debug assert(false, e.msg); - } -} - -/// Specifies the log level for a particular log message. -enum LogLevel { - Trace, - Debug, - Info, - Warn, - Error, - Fatal, - None -} - diff --git a/source/vibecompat/data/json.d b/source/vibecompat/data/json.d deleted file mode 100644 index 1711253..0000000 --- a/source/vibecompat/data/json.d +++ /dev/null @@ -1,1287 +0,0 @@ -/** - JSON serialization and value handling. - - This module provides the Json struct for reading, writing and manipulating JSON values in a seamless, - JavaScript like way. De(serialization) of arbitrary D types is also supported. - - Examples: - - --- - void manipulateJson(Json j) - { - // object members can be accessed using member syntax, just like in JavaScript - j = Json.EmptyObject; - j.name = "Example"; - j.id = 1; - - // retrieving the values is done using get() - assert(j["name"].get!string == "Example"); - assert(j["id"].get!int == 1); - - // semantic convertions can be done using to() - assert(j.id.to!string == "1"); - - // prints: - // name: "Example" - // id: 1 - foreach( string key, value; j ){ - writefln("%s: %s", key, value); - } - - // print out as JSON: {"name": "Example", "id": 1} - writefln("JSON: %s", j.toString()); - } - --- - - Copyright: © 2012 RejectedSoftware e.K. - License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. - Authors: Sönke Ludwig -*/ -module vibecompat.data.json; - -import vibecompat.data.utils; - -import std.array; -import std.conv; -import std.datetime; -import std.exception; -import std.format; -import std.string; -import std.range; -import std.traits; - - -/******************************************************************************/ -/* public types */ -/******************************************************************************/ - -/** - Represents a single JSON value. - - Json values can have one of the types defined in the Json.Type enum. They - behave mostly like values in ECMA script in the way that you can - transparently perform operations on them. However, strict typechecking is - done, so that operations between differently typed JSON values will throw - an exception. Additionally, an explicit cast or using get!() or to!() is - required to convert a JSON value to the corresponding static D type. -*/ -struct Json { - private { - union { - bool m_bool; - long m_int; - double m_float; - string m_string; - Json[] m_array; - Json[string] m_object; - }; - Type m_type = Type.Undefined; - } - - /** Represents the run time type of a JSON value. - */ - enum Type { - /// A non-existent value in a JSON object - Undefined, - /// Null value - Null, - /// Boolean value - Bool, - /// 64-bit integer value - Int, - /// 64-bit floating point value - Float, - /// UTF-8 string - String, - /// Array of JSON values - Array, - /// JSON object aka. dictionary from string to Json - Object - } - - /// New JSON value of Type.Undefined - static @property Json Undefined() { return Json(); } - - /// New JSON value of Type.Object - static @property Json EmptyObject() { return Json(cast(Json[string])null); } - - /// New JSON value of Type.Array - static @property Json EmptyArray() { return Json(cast(Json[])null); } - - version(JsonLineNumbers) int line; - - /** - Constructor for a JSON object. - */ - this(typeof(null)) { m_type = Type.Null; } - /// ditto - this(bool v) { m_type = Type.Bool; m_bool = v; } - /// ditto - this(int v) { m_type = Type.Int; m_int = v; } - /// ditto - this(long v) { m_type = Type.Int; m_int = v; } - /// ditto - this(double v) { m_type = Type.Float; m_float = v; } - /// ditto - this(string v) { m_type = Type.String; m_string = v; } - /// ditto - this(Json[] v) { m_type = Type.Array; m_array = v; } - /// ditto - this(Json[string] v) { m_type = Type.Object; m_object = v; } - - /** - Allows assignment of D values to a JSON value. - */ - ref Json opAssign(Json v){ - m_type = v.m_type; - final switch(m_type){ - case Type.Undefined: m_string = null; break; - case Type.Null: m_string = null; break; - case Type.Bool: m_bool = v.m_bool; break; - case Type.Int: m_int = v.m_int; break; - case Type.Float: m_float = v.m_float; break; - case Type.String: m_string = v.m_string; break; - case Type.Array: m_array = v.m_array; break; - case Type.Object: m_object = v.m_object; break; - } - return this; - } - /// ditto - void opAssign(typeof(null)) { m_type = Type.Null; m_string = null; } - /// ditto - bool opAssign(bool v) { m_type = Type.Bool; m_bool = v; return v; } - /// ditto - int opAssign(int v) { m_type = Type.Int; m_int = v; return v; } - /// ditto - long opAssign(long v) { m_type = Type.Int; m_int = v; return v; } - /// ditto - double opAssign(double v) { m_type = Type.Float; m_float = v; return v; } - /// ditto - string opAssign(string v) { m_type = Type.String; m_string = v; return v; } - /// ditto - Json[] opAssign(Json[] v) { m_type = Type.Array; m_array = v; return v; } - /// ditto - Json[string] opAssign(Json[string] v) { m_type = Type.Object; m_object = v; return v; } - - /** - The current type id of this JSON object. - */ - @property Type type() const { return m_type; } - - /** - Allows direct indexing of array typed JSON values. - */ - ref inout(Json) opIndex(size_t idx) inout { checkType!(Json[])(); return m_array[idx]; } - - /** - Allows direct indexing of object typed JSON values using a string as - the key. - */ - const(Json) opIndex(string key) const { - checkType!(Json[string])(); - if( auto pv = key in m_object ) return *pv; - Json ret = Json.Undefined; - ret.m_string = key; - return ret; - } - /// ditto - ref Json opIndex(string key){ - checkType!(Json[string])(); - if( auto pv = key in m_object ) - return *pv; - m_object[key] = Json(); - m_object[key].m_type = Type.Undefined; // DMDBUG: AAs are teh $H1T!!!11 - assert(m_object[key].type == Type.Undefined); - m_object[key].m_string = key; - return m_object[key]; - } - - /** - Returns a slice of a JSON array. - */ - inout(Json[]) opSlice() inout { checkType!(Json[])(); return m_array; } - /// - inout(Json[]) opSlice(size_t from, size_t to) inout { checkType!(Json[])(); return m_array[from .. to]; } - - /** - Removes an entry from an object. - */ - void remove(string item) { checkType!(Json[string])(); m_object.remove(item); } - - /** - Returns the number of entries of string, array or object typed JSON values. - */ - @property size_t length() - const { - switch(m_type){ - case Type.String: return m_string.length; - case Type.Array: return m_array.length; - case Type.Object: return m_object.length; - default: - enforce(false, "Json.length() can only be called on strings, arrays and objects, not "~.to!string(m_type)~"."); - return 0; - } - } - - /** - Allows foreach iterating over JSON objects and arrays. - */ - int opApply(int delegate(ref Json obj) del) - { - enforce(m_type == Type.Array || m_type == Type.Object, "opApply may only be called on objects and arrays, not "~.to!string(m_type)~"."); - if( m_type == Type.Array ){ - foreach( ref v; m_array ) - if( auto ret = del(v) ) - return ret; - return 0; - } else { - foreach( ref v; m_object ) - if( v.type != Type.Undefined ) - if( auto ret = del(v) ) - return ret; - return 0; - } - } - /// ditto - int opApply(int delegate(ref const Json obj) del) - const { - enforce(m_type == Type.Array || m_type == Type.Object, "opApply may only be called on objects and arrays, not "~.to!string(m_type)~"."); - if( m_type == Type.Array ){ - foreach( ref v; m_array ) - if( auto ret = del(v) ) - return ret; - return 0; - } else { - foreach( ref v; m_object ) - if( v.type != Type.Undefined ) - if( auto ret = del(v) ) - return ret; - return 0; - } - } - /// ditto - int opApply(int delegate(ref size_t idx, ref Json obj) del) - { - enforce(m_type == Type.Array, "opApply may only be called on arrays, not "~.to!string(m_type)~""); - foreach( idx, ref v; m_array ) - if( auto ret = del(idx, v) ) - return ret; - return 0; - } - /// ditto - int opApply(int delegate(ref size_t idx, ref const Json obj) del) - const { - enforce(m_type == Type.Array, "opApply may only be called on arrays, not "~.to!string(m_type)~"."); - foreach( idx, ref v; m_array ) - if( auto ret = del(idx, v) ) - return ret; - return 0; - } - /// ditto - int opApply(int delegate(ref string idx, ref Json obj) del) - { - enforce(m_type == Type.Object, "opApply may only be called on objects, not "~.to!string(m_type)~"."); - foreach( idx, ref v; m_object ) - if( v.type != Type.Undefined ) - if( auto ret = del(idx, v) ) - return ret; - return 0; - } - /// ditto - int opApply(int delegate(ref string idx, ref const Json obj) del) - const { - enforce(m_type == Type.Object, "opApply may only be called on objects, not "~.to!string(m_type)~"."); - foreach( idx, ref v; m_object ) - if( v.type != Type.Undefined ) - if( auto ret = del(idx, v) ) - return ret; - return 0; - } - - /** - Converts the JSON value to the corresponding D type - types must match exactly. - */ - inout(T) opCast(T)() inout { return get!T; } - /// ditto - @property inout(T) get(T)() - inout { - checkType!T(); - static if( is(T == bool) ) return m_bool; - else static if( is(T == double) ) return m_float; - else static if( is(T == float) ) return cast(T)m_float; - else static if( is(T == long) ) return m_int; - else static if( is(T : long) ){ enforce(m_int <= T.max && m_int >= T.min); return cast(T)m_int; } - else static if( is(T == string) ) return m_string; - else static if( is(T == Json[]) ) return m_array; - else static if( is(T == Json[string]) ) return m_object; - else static assert("JSON can only be casted to (bool, long, double, string, Json[] or Json[string]. Not "~T.stringof~"."); - } - /// ditto - @property const(T) opt(T)(const(T) def = T.init) - const { - if( typeId!T != m_type ) return def; - return get!T; - } - /// ditto - @property T opt(T)(T def = T.init) - { - if( typeId!T != m_type ) return def; - return get!T; - } - - /** - Converts the JSON value to the corresponding D type - types are converted as neccessary. - */ - @property inout(T) to(T)() - inout { - static if( is(T == bool) ){ - final switch( m_type ){ - case Type.Undefined: return false; - case Type.Null: return false; - case Type.Bool: return m_bool; - case Type.Int: return m_int != 0; - case Type.Float: return m_float != 0; - case Type.String: return m_string.length > 0; - case Type.Array: return m_array.length > 0; - case Type.Object: return m_object.length > 0; - } - } else static if( is(T == double) ){ - final switch( m_type ){ - case Type.Undefined: return T.init; - case Type.Null: return 0; - case Type.Bool: return m_bool ? 1 : 0; - case Type.Int: return m_int; - case Type.Float: return m_float; - case Type.String: return .to!double(cast(string)m_string); - case Type.Array: return double.init; - case Type.Object: return double.init; - } - } else static if( is(T == float) ){ - final switch( m_type ){ - case Type.Undefined: return T.init; - case Type.Null: return 0; - case Type.Bool: return m_bool ? 1 : 0; - case Type.Int: return m_int; - case Type.Float: return m_float; - case Type.String: return .to!float(cast(string)m_string); - case Type.Array: return float.init; - case Type.Object: return float.init; - } - } - else static if( is(T == long) ){ - final switch( m_type ){ - case Type.Undefined: return 0; - case Type.Null: return 0; - case Type.Bool: return m_bool ? 1 : 0; - case Type.Int: return m_int; - case Type.Float: return cast(long)m_float; - case Type.String: return .to!long(m_string); - case Type.Array: return 0; - case Type.Object: return 0; - } - } else static if( is(T : long) ){ - final switch( m_type ){ - case Type.Undefined: return 0; - case Type.Null: return 0; - case Type.Bool: return m_bool ? 1 : 0; - case Type.Int: return cast(T)m_int; - case Type.Float: return cast(T)m_float; - case Type.String: return cast(T).to!long(cast(string)m_string); - case Type.Array: return 0; - case Type.Object: return 0; - } - } else static if( is(T == string) ){ - switch( m_type ){ - default: return toString(); - case Type.String: return m_string; - } - } else static if( is(T == Json[]) ){ - switch( m_type ){ - default: return Json([this]); - case Type.Array: return m_array; - } - } else static if( is(T == Json[string]) ){ - switch( m_type ){ - default: return Json(["value": this]); - case Type.Object: return m_object; - } - } else static assert("JSON can only be casted to (bool, long, double, string, Json[] or Json[string]. Not "~T.stringof~"."); - } - - /** - Performs unary operations on the JSON value. - - The following operations are supported for each type: - - $(DL - $(DT Null) $(DD none) - $(DT Bool) $(DD ~) - $(DT Int) $(DD +, -, ++, --) - $(DT Float) $(DD +, -, ++, --) - $(DT String) $(DD none) - $(DT Array) $(DD none) - $(DT Object) $(DD none) - ) - */ - Json opUnary(string op)() - const { - static if( op == "~" ){ - checkType!bool(); - return Json(~m_bool); - } else static if( op == "+" || op == "-" || op == "++" || op == "--" ){ - if( m_type == Type.Int ) mixin("return Json("~op~"m_int);"); - else if( m_type == Type.Float ) mixin("return Json("~op~"m_float);"); - else enforce(false, "'"~op~"' only allowed on scalar types, not on "~.to!string(m_type)~"."); - } else static assert("Unsupported operator '"~op~"' for type JSON."); - } - - /** - Performs binary operations between JSON values. - - The two JSON values must be of the same run time type or an exception - will be thrown. Only the operations listed are allowed for each of the - types. - - $(DL - $(DT Null) $(DD none) - $(DT Bool) $(DD &&, ||) - $(DT Int) $(DD +, -, *, /, %) - $(DT Float) $(DD +, -, *, /, %) - $(DT String) $(DD ~) - $(DT Array) $(DD ~) - $(DT Object) $(DD none) - ) - */ - Json opBinary(string op)(ref const(Json) other) - const { - enforce(m_type == other.m_type, "Binary operation '"~op~"' between "~.to!string(m_type)~" and "~.to!string(other.m_type)~" JSON objects."); - static if( op == "&&" ){ - enforce(m_type == Type.Bool, "'&&' only allowed for Type.Bool, not "~.to!string(m_type)~"."); - return Json(m_bool && other.m_bool); - } else static if( op == "||" ){ - enforce(m_type == Type.Bool, "'||' only allowed for Type.Bool, not "~.to!string(m_type)~"."); - return Json(m_bool || other.m_bool); - } else static if( op == "+" ){ - if( m_type == Type.Int ) return Json(m_int + other.m_int); - else if( m_type == Type.Float ) return Json(m_float + other.m_float); - else enforce(false, "'+' only allowed for scalar types, not "~.to!string(m_type)~"."); - } else static if( op == "-" ){ - if( m_type == Type.Int ) return Json(m_int - other.m_int); - else if( m_type == Type.Float ) return Json(m_float - other.m_float); - else enforce(false, "'-' only allowed for scalar types, not "~.to!string(m_type)~"."); - } else static if( op == "*" ){ - if( m_type == Type.Int ) return Json(m_int * other.m_int); - else if( m_type == Type.Float ) return Json(m_float * other.m_float); - else enforce(false, "'*' only allowed for scalar types, not "~.to!string(m_type)~"."); - } else static if( op == "/" ){ - if( m_type == Type.Int ) return Json(m_int / other.m_int); - else if( m_type == Type.Float ) return Json(m_float / other.m_float); - else enforce(false, "'/' only allowed for scalar types, not "~.to!string(m_type)~"."); - } else static if( op == "%" ){ - if( m_type == Type.Int ) return Json(m_int % other.m_int); - else if( m_type == Type.Float ) return Json(m_float % other.m_float); - else enforce(false, "'%' only allowed for scalar types, not "~.to!string(m_type)~"."); - } else static if( op == "~" ){ - if( m_type == Type.String ) return Json(m_string ~ other.m_string); - else enforce(false, "'~' only allowed for strings, not "~.to!string(m_type)~"."); - } else static assert("Unsupported operator '"~op~"' for type JSON."); - assert(false); - } - /// ditto - Json opBinary(string op)(Json other) - if( op == "~" ) - { - static if( op == "~" ){ - if( m_type == Type.String ) return Json(m_string ~ other.m_string); - else if( m_type == Type.Array ) return Json(m_array ~ other.m_array); - else enforce(false, "'~' only allowed for strings and arrays, not "~.to!string(m_type)~"."); - } else static assert("Unsupported operator '"~op~"' for type JSON."); - assert(false); - } - /// ditto - void opOpAssign(string op)(Json other) - if( op == "+" || op == "-" || op == "*" ||op == "/" || op == "%" ) - { - enforce(m_type == other.m_type, "Binary operation '"~op~"' between "~.to!string(m_type)~" and "~.to!string(other.m_type)~" JSON objects."); - static if( op == "+" ){ - if( m_type == Type.Int ) m_int += other.m_int; - else if( m_type == Type.Float ) m_float += other.m_float; - else enforce(false, "'+' only allowed for scalar types, not "~.to!string(m_type)~"."); - } else static if( op == "-" ){ - if( m_type == Type.Int ) m_int -= other.m_int; - else if( m_type == Type.Float ) m_float -= other.m_float; - else enforce(false, "'-' only allowed for scalar types, not "~.to!string(m_type)~"."); - } else static if( op == "*" ){ - if( m_type == Type.Int ) m_int *= other.m_int; - else if( m_type == Type.Float ) m_float *= other.m_float; - else enforce(false, "'*' only allowed for scalar types, not "~.to!string(m_type)~"."); - } else static if( op == "/" ){ - if( m_type == Type.Int ) m_int /= other.m_int; - else if( m_type == Type.Float ) m_float /= other.m_float; - else enforce(false, "'/' only allowed for scalar types, not "~.to!string(m_type)~"."); - } else static if( op == "%" ){ - if( m_type == Type.Int ) m_int %= other.m_int; - else if( m_type == Type.Float ) m_float %= other.m_float; - else enforce(false, "'%' only allowed for scalar types, not "~.to!string(m_type)~"."); - } /*else static if( op == "~" ){ - if( m_type == Type.String ) m_string ~= other.m_string; - else if( m_type == Type.Array ) m_array ~= other.m_array; - else enforce(false, "'%' only allowed for scalar types, not "~.to!string(m_type)~"."); - }*/ else static assert("Unsupported operator '"~op~"' for type JSON."); - assert(false); - } - /// ditto - Json opBinary(string op)(bool other) const { checkType!bool(); mixin("return Json(m_bool "~op~" other);"); } - /// ditto - Json opBinary(string op)(long other) const { checkType!long(); mixin("return Json(m_int "~op~" other);"); } - /// ditto - Json opBinary(string op)(double other) const { checkType!double(); mixin("return Json(m_float "~op~" other);"); } - /// ditto - Json opBinary(string op)(string other) const { checkType!string(); mixin("return Json(m_string "~op~" other);"); } - /// ditto - Json opBinary(string op)(Json[] other) { checkType!(Json[])(); mixin("return Json(m_array "~op~" other);"); } - /// ditto - Json opBinaryRight(string op)(bool other) const { checkType!bool(); mixin("return Json(other "~op~" m_bool);"); } - /// ditto - Json opBinaryRight(string op)(long other) const { checkType!long(); mixin("return Json(other "~op~" m_int);"); } - /// ditto - Json opBinaryRight(string op)(double other) const { checkType!double(); mixin("return Json(other "~op~" m_float);"); } - /// ditto - Json opBinaryRight(string op)(string other) const if(op == "~") { checkType!string(); return Json(other ~ m_string); } - /// ditto - inout(Json)* opBinaryRight(string op)(string other) inout if(op == "in") { - checkType!(Json[string])(); - auto pv = other in m_object; - if( !pv ) return null; - if( pv.type == Type.Undefined ) return null; - return pv; - } - /// ditto - Json opBinaryRight(string op)(Json[] other) { checkType!(Json[])(); mixin("return Json(other "~op~" m_array);"); } - - /** - Allows to access existing fields of a JSON object using dot syntax. - */ - @property const(Json) opDispatch(string prop)() const { return opIndex(prop); } - /// ditto - @property ref Json opDispatch(string prop)() { return opIndex(prop); } - - /** - Compares two JSON values for equality. - - If the two values have different types, they are considered unequal. - This differs with ECMA script, which performs a type conversion before - comparing the values. - */ - bool opEquals(ref const Json other) - const { - if( m_type != other.m_type ) return false; - final switch(m_type){ - case Type.Undefined: return false; - case Type.Null: return true; - case Type.Bool: return m_bool == other.m_bool; - case Type.Int: return m_int == other.m_int; - case Type.Float: return m_float == other.m_float; - case Type.String: return m_string == other.m_string; - case Type.Array: return m_array == other.m_array; - case Type.Object: return m_object == other.m_object; - } - } - /// ditto - bool opEquals(const Json other) const { return opEquals(other); } - /// ditto - bool opEquals(typeof(null)) const { return m_type == Type.Null; } - /// ditto - bool opEquals(bool v) const { return m_type == Type.Bool && m_bool == v; } - /// ditto - bool opEquals(long v) const { return m_type == Type.Int && m_int == v; } - /// ditto - bool opEquals(double v) const { return m_type == Type.Float && m_float == v; } - /// ditto - bool opEquals(string v) const { return m_type == Type.String && m_string == v; } - - /** - Compares two JSON values. - - If the types of the two values differ, the value with the smaller type - id is considered the smaller value. This differs from ECMA script, which - performs a type conversion before comparing the values. - - JSON values of type Object cannot be compared and will throw an - exception. - */ - int opCmp(ref const Json other) - const { - if( m_type != other.m_type ) return m_type < other.m_type ? -1 : 1; - final switch(m_type){ - case Type.Undefined: return 0; - case Type.Null: return 0; - case Type.Bool: return m_bool < other.m_bool ? -1 : m_bool == other.m_bool ? 0 : 1; - case Type.Int: return m_int < other.m_int ? -1 : m_int == other.m_int ? 0 : 1; - case Type.Float: return m_float < other.m_float ? -1 : m_float == other.m_float ? 0 : 1; - case Type.String: return m_string < other.m_string ? -1 : m_string == other.m_string ? 0 : 1; - case Type.Array: return m_array < other.m_array ? -1 : m_array == other.m_array ? 0 : 1; - case Type.Object: - enforce(false, "JSON objects cannot be compared."); - assert(false); - } - } - - - - /** - Returns the type id corresponding to the given D type. - */ - static @property Type typeId(T)() { - static if( is(T == typeof(null)) ) return Type.Null; - else static if( is(T == bool) ) return Type.Bool; - else static if( is(T == double) ) return Type.Float; - else static if( is(T == float) ) return Type.Float; - else static if( is(T : long) ) return Type.Int; - else static if( is(T == string) ) return Type.String; - else static if( is(T == Json[]) ) return Type.Array; - else static if( is(T == Json[string]) ) return Type.Object; - else static assert(false, "Unsupported JSON type '"~T.stringof~"'. Only bool, long, double, string, Json[] and Json[string] are allowed."); - } - - /** - Returns the JSON object as a string. - - For large JSON values use writeJsonString instead as this function will store the whole string - in memory, whereas writeJsonString writes it out bit for bit. - - See_Also: writeJsonString, toPrettyString - */ - string toString() - const { - auto ret = appender!string(); - writeJsonString(ret, this); - return ret.data; - } - - /** - Returns the JSON object as a "pretty" string. - - --- - auto json = Json(["foo": Json("bar")]); - writeln(json.toPrettyString()); - - // output: - // { - // "foo": "bar" - // } - --- - - Params: - level = Specifies the base amount of indentation for the output. Indentation is always - done using tab characters. - - See_Also: writePrettyJsonString, toString - */ - string toPrettyString(int level = 0) - const { - auto ret = appender!string(); - writePrettyJsonString(ret, this, level); - return ret.data; - } - - private void checkType(T)() - const { - string dbg; - if( m_type == Type.Undefined ) dbg = " field "~m_string; - enforce(typeId!T == m_type, "Trying to access JSON"~dbg~" of type "~.to!string(m_type)~" as "~T.stringof~"."); - } - - /*invariant() - { - assert(m_type >= Type.Undefined && m_type <= Type.Object); - }*/ -} - - -/******************************************************************************/ -/* public functions */ -/******************************************************************************/ - -/** - Parses the given range as a JSON string and returns the corresponding Json object. - - The range is shrunk during parsing, leaving any remaining text that is not part of - the JSON contents. - - Throws an Exception if any parsing error occured. -*/ -Json parseJson(R)(ref R range, int* line = null) - if( is(R == string) ) -{ - Json ret; - enforce(!range.empty, "JSON string is empty."); - - skipWhitespace(range, line); - - version(JsonLineNumbers){ - import vibecompat.core.log; - int curline = line ? *line : 0; - scope(failure) logError("Error in line: %d", curline); - } - - switch( range.front ){ - case 'f': - enforce(range[1 .. $].startsWith("alse"), "Expected 'false', got '"~range[0 .. 5]~"'."); - range.popFrontN(5); - ret = false; - break; - case 'n': - enforce(range[1 .. $].startsWith("ull"), "Expected 'null', got '"~range[0 .. 4]~"'."); - range.popFrontN(4); - ret = null; - break; - case 't': - enforce(range[1 .. $].startsWith("rue"), "Expected 'true', got '"~range[0 .. 4]~"'."); - range.popFrontN(4); - ret = true; - break; - case '0': .. case '9'+1: - case '-': - bool is_float; - auto num = skipNumber(range, is_float); - if( is_float ) ret = to!double(num); - else ret = to!long(num); - break; - case '\"': - ret = skipJsonString(range); - break; - case '[': - Json[] arr; - range.popFront(); - while(true) { - skipWhitespace(range, line); - enforce(!range.empty); - if(range.front == ']') break; - arr ~= parseJson(range, line); - skipWhitespace(range, line); - enforce(!range.empty && (range.front == ',' || range.front == ']'), "Expected ']' or ','."); - if( range.front == ']' ) break; - else range.popFront(); - } - range.popFront(); - ret = arr; - break; - case '{': - Json[string] obj; - range.popFront(); - while(true) { - skipWhitespace(range, line); - enforce(!range.empty); - if(range.front == '}') break; - string key = skipJsonString(range); - skipWhitespace(range, line); - enforce(range.startsWith(":"), "Expected ':' for key '" ~ key ~ "'"); - range.popFront(); - skipWhitespace(range, line); - Json itm = parseJson(range, line); - obj[key] = itm; - skipWhitespace(range, line); - enforce(!range.empty && (range.front == ',' || range.front == '}'), "Expected '}' or ',' - got '"~range[0]~"'."); - if( range.front == '}' ) break; - else range.popFront(); - } - range.popFront(); - ret = obj; - break; - default: - enforce(false, "Expected valid json token, got '"~to!string(range.length)~range[0 .. range.length>12?12:range.length]~"'."); - } - - assert(ret.type != Json.Type.Undefined); - version(JsonLineNumbers) ret.line = curline; - return ret; -} - -/** - Parses the given JSON string and returns the corresponding Json object. - - Throws an Exception if any parsing error occurs. -*/ -Json parseJsonString(string str) -{ - auto ret = parseJson(str); - enforce(str.strip().length == 0, "Expected end of string after JSON value."); - return ret; -} - -unittest { - assert(parseJsonString("null") == Json(null)); - assert(parseJsonString("true") == Json(true)); - assert(parseJsonString("false") == Json(false)); - assert(parseJsonString("1") == Json(1)); - assert(parseJsonString("2.0") == Json(2.0)); - assert(parseJsonString("\"test\"") == Json("test")); - assert(parseJsonString("[1, 2, 3]") == Json([Json(1), Json(2), Json(3)])); - assert(parseJsonString("{\"a\": 1}") == Json(["a": Json(1)])); - assert(parseJsonString(`"\\\/\b\f\n\r\t\u1234"`).get!string == "\\/\b\f\n\r\t\u1234"); -} - - -/** - Serializes the given value to JSON. - - The following types of values are supported: - - $(DL - $(DT Json) $(DD Used as-is) - $(DT null) $(DD Converted to Json.Type.Null) - $(DT bool) $(DD Converted to Json.Type.Bool) - $(DT float, double) $(DD Converted to Json.Type.Double) - $(DT short, ushort, int, uint, long, ulong) $(DD Converted to Json.Type.Int) - $(DT string) $(DD Converted to Json.Type.String) - $(DT T[]) $(DD Converted to Json.Type.Array) - $(DT T[string]) $(DD Converted to Json.Type.Object) - $(DT struct) $(DD Converted to Json.Type.Object) - $(DT class) $(DD Converted to Json.Type.Object or Json.Type.Null) - ) - - All entries of an array or an associative array, as well as all R/W properties and - all public fields of a struct/class are recursively serialized using the same rules. - - Fields ending with an underscore will have the last underscore stripped in the - serialized output. This makes it possible to use fields with D keywords as their name - by simply appending an underscore. - - The following methods can be used to customize the serialization of structs/classes: - - --- - Json toJson() const; - static T fromJson(Json src); - - string toString() const; - static T fromString(string src); - --- - - The methods will have to be defined in pairs. The first pair that is implemented by - the type will be used for serialization (i.e. toJson overrides toString). -*/ -Json serializeToJson(T)(T value) -{ - alias Unqual!T TU; - static if( is(TU == Json) ) return value; - else static if( is(TU == typeof(null)) ) return Json(null); - else static if( is(TU == bool) ) return Json(value); - else static if( is(TU == float) ) return Json(cast(double)value); - else static if( is(TU == double) ) return Json(value); - else static if( is(TU == DateTime) ) return Json(value.toISOExtString()); - else static if( is(TU == SysTime) ) return Json(value.toISOExtString()); - else static if( is(TU : long) ) return Json(cast(long)value); - else static if( is(TU == string) ) return Json(value); - else static if( isArray!T ){ - auto ret = new Json[value.length]; - foreach( i; 0 .. value.length ) - ret[i] = serializeToJson(value[i]); - return Json(ret); - } else static if( isAssociativeArray!TU ){ - Json[string] ret; - foreach( string key, value; value ) - ret[key] = serializeToJson(value); - return Json(ret); - } else static if( __traits(compiles, value = T.fromJson(value.toJson())) ){ - return value.toJson(); - } else static if( __traits(compiles, value = T.fromString(value.toString())) ){ - return Json(value.toString()); - } else static if( is(TU == struct) ){ - Json[string] ret; - foreach( m; __traits(allMembers, T) ){ - static if( isRWField!(TU, m) ){ - auto mv = __traits(getMember, value, m); - ret[underscoreStrip(m)] = serializeToJson(mv); - } - } - return Json(ret); - } else static if( is(TU == class) ){ - if( value is null ) return Json(null); - Json[string] ret; - foreach( m; __traits(allMembers, T) ){ - static if( isRWField!(TU, m) ){ - auto mv = __traits(getMember, value, m); - ret[underscoreStrip(m)] = serializeToJson(mv); - } - } - return Json(ret); - } else static if( isPointer!TU ){ - if( value is null ) return Json(null); - return serializeToJson(*value); - } else { - static assert(false, "Unsupported type '"~T.stringof~"' for JSON serialization."); - } -} - - -/** - Deserializes a JSON value into the destination variable. - - The same types as for serializeToJson() are supported and handled inversely. -*/ -void deserializeJson(T)(ref T dst, Json src) -{ - dst = deserializeJson!T(src); -} -/// ditto -T deserializeJson(T)(Json src) -{ - static if( is(T == Json) ) return src; - else static if( is(T == typeof(null)) ){ return null; } - else static if( is(T == bool) ) return src.get!bool; - else static if( is(T == float) ) return src.to!float; // since doubles are frequently serialized without - else static if( is(T == double) ) return src.to!double; // a decimal point, we allow conversions here - else static if( is(T == DateTime) ) return DateTime.fromISOExtString(src.get!string); - else static if( is(T == SysTime) ) return SysTime.fromISOExtString(src.get!string); - else static if( is(T : long) ) return cast(T)src.get!long; - else static if( is(T == string) ) return src.get!string; - else static if( isArray!T ){ - alias typeof(T.init[0]) TV; - auto dst = new Unqual!TV[src.length]; - foreach( size_t i, v; src ) - dst[i] = deserializeJson!(Unqual!TV)(v); - return dst; - } else static if( isAssociativeArray!T ){ - alias typeof(T.init.values[0]) TV; - Unqual!TV[string] dst; - foreach( string key, value; src ) - dst[key] = deserializeJson!(Unqual!TV)(value); - return dst; - } else static if( __traits(compiles, { T dst; dst = T.fromJson(dst.toJson()); }()) ){ - return T.fromJson(src); - } else static if( __traits(compiles, { T dst; dst = T.fromString(dst.toString()); }()) ){ - return T.fromString(src.get!string); - } else static if( is(T == struct) ){ - T dst; - foreach( m; __traits(allMembers, T) ){ - static if( isRWPlainField!(T, m) || isRWField!(T, m) ){ - alias typeof(__traits(getMember, dst, m)) TM; - __traits(getMember, dst, m) = deserializeJson!TM(src[underscoreStrip(m)]); - } - } - return dst; - } else static if( is(T == class) ){ - if( src.type == Json.Type.Null ) return null; - auto dst = new T; - foreach( m; __traits(allMembers, T) ){ - static if( isRWPlainField!(T, m) || isRWField!(T, m) ){ - alias typeof(__traits(getMember, dst, m)) TM; - __traits(getMember, dst, m) = deserializeJson!TM(src[underscoreStrip(m)]); - } - } - return dst; - } else static if( isPointer!T ){ - if( src.type == Json.Type.Null ) return null; - alias typeof(*T.init) TD; - dst = new TD; - *dst = deserializeJson!TD(src); - return dst; - } else { - static assert(false, "Unsupported type '"~T.stringof~"' for JSON serialization."); - } -} - -unittest { - import std.stdio; - static struct S { float a; double b; bool c; int d; string e; byte f; ubyte g; long h; ulong i; float[] j; } - immutable S t = {1.5, -3.0, true, int.min, "Test", -128, 255, long.min, ulong.max, [1.1, 1.2, 1.3]}; - S u; - deserializeJson(u, serializeToJson(t)); - assert(t.a == u.a); - assert(t.b == u.b); - assert(t.c == u.c); - assert(t.d == u.d); - assert(t.e == u.e); - assert(t.f == u.f); - assert(t.g == u.g); - assert(t.h == u.h); - assert(t.i == u.i); - assert(t.j == u.j); -} - -unittest { - static class C { - int a; - private int _b; - @property int b() const { return _b; } - @property void b(int v) { _b = v; } - - @property int test() const { return 10; } - - void test2() {} - } - C c = new C; - c.a = 1; - c.b = 2; - - C d; - deserializeJson(d, serializeToJson(c)); - assert(c.a == d.a); - assert(c.b == d.b); -} - - -/** - Writes the given JSON object as a JSON string into the destination range. - - This function will convert the given JSON value to a string without adding - any white space between tokens (no newlines, no indentation and no padding). - The output size is thus minizized, at the cost of bad human readability. - - Params: - dst = References the string output range to which the result is written. - json = Specifies the JSON value that is to be stringified. - - See_Also: Json.toString, writePrettyJsonString -*/ -void writeJsonString(R)(ref R dst, in Json json) -// if( isOutputRange!R && is(ElementEncodingType!R == char) ) -{ - final switch( json.type ){ - case Json.Type.Undefined: dst.put("undefined"); break; - case Json.Type.Null: dst.put("null"); break; - case Json.Type.Bool: dst.put(cast(bool)json ? "true" : "false"); break; - case Json.Type.Int: formattedWrite(dst, "%d", json.get!long); break; - case Json.Type.Float: formattedWrite(dst, "%.16g", json.get!double); break; - case Json.Type.String: - dst.put("\""); - jsonEscape(dst, cast(string)json); - dst.put("\""); - break; - case Json.Type.Array: - dst.put("["); - bool first = true; - foreach( ref const Json e; json ){ - if( e.type == Json.Type.Undefined ) continue; - if( !first ) dst.put(","); - first = false; - writeJsonString(dst, e); - } - dst.put("]"); - break; - case Json.Type.Object: - dst.put("{"); - bool first = true; - foreach( string k, ref const Json e; json ){ - if( e.type == Json.Type.Undefined ) continue; - if( !first ) dst.put(","); - first = false; - dst.put("\""); - jsonEscape(dst, k); - dst.put("\":"); - writeJsonString(dst, e); - } - dst.put("}"); - break; - } -} - -/** - Writes the given JSON object as a prettified JSON string into the destination range. - - The output will contain newlines and indents to make the output human readable. - - Params: - dst = References the string output range to which the result is written. - json = Specifies the JSON value that is to be stringified. - level = Specifies the base amount of indentation for the output. Indentation is always - done using tab characters. - - See_Also: Json.toPrettyString, writeJsonString -*/ -void writePrettyJsonString(R)(ref R dst, in Json json, int level = 0) -// if( isOutputRange!R && is(ElementEncodingType!R == char) ) -{ - final switch( json.type ){ - case Json.Type.Undefined: dst.put("undefined"); break; - case Json.Type.Null: dst.put("null"); break; - case Json.Type.Bool: dst.put(cast(bool)json ? "true" : "false"); break; - case Json.Type.Int: formattedWrite(dst, "%d", json.get!long); break; - case Json.Type.Float: formattedWrite(dst, "%.16g", json.get!double); break; - case Json.Type.String: - dst.put("\""); - jsonEscape(dst, cast(string)json); - dst.put("\""); - break; - case Json.Type.Array: - dst.put("["); - bool first = true; - foreach( e; json ){ - if( e.type == Json.Type.Undefined ) continue; - if( !first ) dst.put(","); - first = false; - dst.put("\n"); - foreach( tab; 0 .. level ) dst.put('\t'); - writePrettyJsonString(dst, e, level+1); - } - if( json.length > 0 ) { - dst.put('\n'); - foreach( tab; 0 .. (level-1) ) dst.put('\t'); - } - dst.put("]"); - break; - case Json.Type.Object: - dst.put("{"); - bool first = true; - foreach( string k, e; json ){ - if( e.type == Json.Type.Undefined ) continue; - if( !first ) dst.put(","); - dst.put("\n"); - first = false; - foreach( tab; 0 .. level ) dst.put('\t'); - dst.put("\""); - jsonEscape(dst, k); - dst.put("\": "); - writePrettyJsonString(dst, e, level+1); - } - if( json.length > 0 ) { - dst.put('\n'); - foreach( tab; 0 .. (level-1) ) dst.put('\t'); - } - dst.put("}"); - break; - } -} - - -/** Deprecated aliases for backwards compatibility. - - Use writeJsonString and writePrettyJsonString instead. -*/ -deprecated("Please use writeJsonString instead.") alias writeJsonString toJson; -/// -deprecated("Please use writePrettyJsonString instead.") alias writePrettyJsonString toPrettyJson; - - -/// private -private void jsonEscape(R)(ref R dst, string s) -{ - foreach( ch; s ){ - switch(ch){ - default: dst.put(ch); break; - case '\\': dst.put("\\\\"); break; - case '\r': dst.put("\\r"); break; - case '\n': dst.put("\\n"); break; - case '\t': dst.put("\\t"); break; - case '\"': dst.put("\\\""); break; - } - } -} - -/// private -private string jsonUnescape(R)(ref R range) -{ - auto ret = appender!string(); - while(!range.empty){ - auto ch = range.front; - switch( ch ){ - case '"': return ret.data; - case '\\': - range.popFront(); - enforce(!range.empty, "Unterminated string escape sequence."); - switch(range.front){ - default: enforce("Invalid string escape sequence."); break; - case '"': ret.put('\"'); range.popFront(); break; - case '\\': ret.put('\\'); range.popFront(); break; - case '/': ret.put('/'); range.popFront(); break; - case 'b': ret.put('\b'); range.popFront(); break; - case 'f': ret.put('\f'); range.popFront(); break; - case 'n': ret.put('\n'); range.popFront(); break; - case 'r': ret.put('\r'); range.popFront(); break; - case 't': ret.put('\t'); range.popFront(); break; - case 'u': - range.popFront(); - dchar uch = 0; - foreach( i; 0 .. 4 ){ - uch *= 16; - enforce(!range.empty, "Unicode sequence must be '\\uXXXX'."); - auto dc = range.front; - range.popFront(); - if( dc >= '0' && dc <= '9' ) uch += dc - '0'; - else if( dc >= 'a' && dc <= 'f' ) uch += dc - 'a' + 10; - else if( dc >= 'A' && dc <= 'F' ) uch += dc - 'A' + 10; - else enforce(false, "Unicode sequence must be '\\uXXXX'."); - } - ret.put(uch); - break; - } - break; - default: - ret.put(ch); - range.popFront(); - break; - } - } - return ret.data; -} - -private string skipNumber(ref string s, out bool is_float) -{ - size_t idx = 0; - is_float = false; - if( s[idx] == '-' ) idx++; - if( s[idx] == '0' ) idx++; - else { - enforce(isDigit(s[idx++]), "Digit expected at beginning of number."); - while( idx < s.length && isDigit(s[idx]) ) idx++; - } - - if( idx < s.length && s[idx] == '.' ){ - idx++; - is_float = true; - while( idx < s.length && isDigit(s[idx]) ) idx++; - } - - if( idx < s.length && (s[idx] == 'e' || s[idx] == 'E') ){ - idx++; - is_float = true; - if( idx < s.length && (s[idx] == '+' || s[idx] == '-') ) idx++; - enforce( idx < s.length && isDigit(s[idx]), "Expected exponent." ~ s[0 .. idx]); - idx++; - while( idx < s.length && isDigit(s[idx]) ) idx++; - } - - string ret = s[0 .. idx]; - s = s[idx .. $]; - return ret; -} - -private string skipJsonString(ref string s, int* line = null) -{ - enforce(s.length >= 2 && s[0] == '\"', "too small: '" ~ s ~ "'"); - s = s[1 .. $]; - string ret = jsonUnescape(s); - enforce(s.length > 0 && s[0] == '\"', "Unterminated string literal."); - s = s[1 .. $]; - return ret; -} - -private void skipWhitespace(ref string s, int* line = null) -{ - while( s.length > 0 ){ - switch( s[0] ){ - default: return; - case ' ', '\t': s = s[1 .. $]; break; - case '\n': - s = s[1 .. $]; - if( s.length > 0 && s[0] == '\r' ) s = s[1 .. $]; - if( line ) (*line)++; - break; - case '\r': - s = s[1 .. $]; - if( s.length > 0 && s[0] == '\n' ) s = s[1 .. $]; - if( line ) (*line)++; - break; - } - } -} - -/// private -private bool isDigit(T)(T ch){ return ch >= '0' && ch <= '9'; } - -private string underscoreStrip(string field_name) -{ - if( field_name.length < 1 || field_name[$-1] != '_' ) return field_name; - else return field_name[0 .. $-1]; -} diff --git a/source/vibecompat/data/utils.d b/source/vibecompat/data/utils.d deleted file mode 100644 index 338dd61..0000000 --- a/source/vibecompat/data/utils.d +++ /dev/null @@ -1,30 +0,0 @@ -/** - Utility functions for data serialization - - Copyright: © 2012 RejectedSoftware e.K. - License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. - Authors: Sönke Ludwig -*/ -module vibecompat.data.utils; - -public import std.traits; - - -template isRWPlainField(T, string M) -{ - static if( !__traits(compiles, typeof(__traits(getMember, T, M))) ){ - enum isRWPlainField = false; - } else { - //pragma(msg, T.stringof~"."~M~":"~typeof(__traits(getMember, T, M)).stringof); - enum isRWPlainField = isRWField!(T, M) && __traits(compiles, *(&__traits(getMember, Tgen!T(), M)) = *(&__traits(getMember, Tgen!T(), M))); - } -} - -template isRWField(T, string M) -{ - enum isRWField = __traits(compiles, __traits(getMember, Tgen!T(), M) = __traits(getMember, Tgen!T(), M)); - //pragma(msg, T.stringof~"."~M~": "~(isRWField?"1":"0")); -} - -/// private -private T Tgen(T)(){ return T.init; } diff --git a/source/vibecompat/inet/path.d b/source/vibecompat/inet/path.d deleted file mode 100644 index 87df835..0000000 --- a/source/vibecompat/inet/path.d +++ /dev/null @@ -1,303 +0,0 @@ -/** - Contains routines for high level path handling. - - Copyright: © 2012 RejectedSoftware e.K. - License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. - Authors: Sönke Ludwig -*/ -module vibecompat.inet.path; - -import std.algorithm; -import std.array; -import std.conv; -import std.exception; -import std.string; - - -/** - Represents an absolute or relative file system path. - - This struct allows to do safe operations on paths, such as concatenation and sub paths. Checks - are done to disallow invalid operations such as concatenating two absolute paths. It also - validates path strings and allows for easy checking of malicious relative paths. -*/ -struct Path { - private { - immutable(PathEntry)[] m_nodes; - bool m_absolute = false; - bool m_endsWithSlash = false; - } - - /// Constructs a Path object by parsing a path string. - this(string pathstr) - { - m_nodes = cast(immutable)splitPath(pathstr); - m_absolute = (pathstr.startsWith("/") || m_nodes.length > 0 && m_nodes[0].toString().countUntil(':')>0); - m_endsWithSlash = pathstr.endsWith("/"); - foreach( e; m_nodes ) assert(e.toString().length > 0); - } - - /// Constructs a path object from a list of PathEntry objects. - this(immutable(PathEntry)[] nodes, bool absolute) - { - m_nodes = nodes; - m_absolute = absolute; - } - - /// Constructs a relative path with one path entry. - this(PathEntry entry){ - m_nodes = [entry]; - m_absolute = false; - } - - /// Determines if the path is absolute. - @property bool absolute() const { return m_absolute; } - - /// Resolves all '.' and '..' path entries as far as possible. - void normalize() - { - immutable(PathEntry)[] newnodes; - foreach( n; m_nodes ){ - switch(n.toString()){ - default: - newnodes ~= n; - break; - case ".": break; - case "..": - enforce(!m_absolute || newnodes.length > 0, "Path goes below root node."); - if( newnodes.length > 0 && newnodes[$-1] != ".." ) newnodes = newnodes[0 .. $-1]; - else newnodes ~= n; - break; - } - } - m_nodes = newnodes; - } - - /// Converts the Path back to a string representation using slashes. - string toString() - const { - if( m_nodes.empty ) return absolute ? "/" : ""; - - Appender!string ret; - - // for absolute paths start with / - if( absolute ) ret.put('/'); - - foreach( i, f; m_nodes ){ - if( i > 0 ) ret.put('/'); - ret.put(f.toString()); - } - - if( m_nodes.length > 0 && m_endsWithSlash ) - ret.put('/'); - - return ret.data; - } - - /// Converts the Path object to a native path string (backslash as path separator on Windows). - string toNativeString() - const { - Appender!string ret; - - // for absolute unix paths start with / - version(Posix) { if(absolute) ret.put('/'); } - - foreach( i, f; m_nodes ){ - version(Windows) { if( i > 0 ) ret.put('\\'); } - version(Posix) { if( i > 0 ) ret.put('/'); } - else { enforce("Unsupported OS"); } - ret.put(f.toString()); - } - - if( m_nodes.length > 0 && m_endsWithSlash ){ - version(Windows) { ret.put('\\'); } - version(Posix) { ret.put('/'); } - } - - return ret.data; - } - - /// Tests if `rhs` is an anchestor or the same as this path. - bool startsWith(const Path rhs) const { - if( rhs.m_nodes.length > m_nodes.length ) return false; - foreach( i; 0 .. rhs.m_nodes.length ) - if( m_nodes[i] != rhs.m_nodes[i] ) - return false; - return true; - } - - /// Computes the relative path from `parentPath` to this path. - Path relativeTo(const Path parentPath) const { - version(Windows){ - // a path such as ..\C:\windows is not valid, so force the path to stay absolute in this case - if( this.absolute && !this.empty && m_nodes[0].toString().endsWith(":") && - !parentPath.startsWith(this[0 .. 1]) ) - { - return this; - } - } - int nup = 0; - while( parentPath.length > nup && !startsWith(parentPath[0 .. parentPath.length-nup]) ){ - nup++; - } - Path ret = Path(null, false); - ret.m_endsWithSlash = true; - foreach( i; 0 .. nup ) ret ~= ".."; - ret ~= Path(m_nodes[parentPath.length-nup .. $], false); - return ret; - } - - /// The last entry of the path - @property ref immutable(PathEntry) head() const { enforce(m_nodes.length > 0); return m_nodes[$-1]; } - - /// The parent path - @property Path parentPath() const { return this[0 .. length-1]; } - - /// The ist of path entries of which this path is composed - @property immutable(PathEntry)[] nodes() const { return m_nodes; } - - /// The number of path entries of which this path is composed - @property size_t length() const { return m_nodes.length; } - - /// True if the path contains no entries - @property bool empty() const { return m_nodes.length == 0; } - - /// Determines if the path ends with a slash (i.e. is a directory) - @property bool endsWithSlash() const { return m_endsWithSlash; } - /// ditto - @property void endsWithSlash(bool v) { m_endsWithSlash = v; } - - /// Determines if this path goes outside of its base path (i.e. begins with '..'). - @property bool external() const { return !m_absolute && m_nodes.length > 0 && m_nodes[0].m_name == ".."; } - - ref immutable(PathEntry) opIndex(size_t idx) const { return m_nodes[idx]; } - Path opSlice(size_t start, size_t end) const { - auto ret = Path(m_nodes[start .. end], start == 0 ? absolute : false); - if( end == m_nodes.length ) ret.m_endsWithSlash = m_endsWithSlash; - return ret; - } - size_t opDollar(int dim)() const if(dim == 0) { return m_nodes.length; } - - - Path opBinary(string OP)(const Path rhs) const if( OP == "~" ) { - Path ret; - ret.m_nodes = m_nodes; - ret.m_absolute = m_absolute; - ret.m_endsWithSlash = rhs.m_endsWithSlash; - ret.normalize(); // needed to avoid "."~".." become "" instead of ".." - - assert(!rhs.absolute, "Trying to append absolute path."); - size_t idx = m_nodes.length; - foreach(folder; rhs.m_nodes){ - switch(folder.toString()){ - default: ret.m_nodes = ret.m_nodes ~ folder; break; - case ".": break; - case "..": - enforce(!ret.absolute || ret.m_nodes.length > 0, "Relative path goes below root node!"); - if( ret.m_nodes.length > 0 && ret.m_nodes[$-1].toString() != ".." ) - ret.m_nodes = ret.m_nodes[0 .. $-1]; - else ret.m_nodes = ret.m_nodes ~ folder; - break; - } - } - return ret; - } - - Path opBinary(string OP)(string rhs) const if( OP == "~" ) { assert(rhs.length > 0); return opBinary!"~"(Path(rhs)); } - Path opBinary(string OP)(PathEntry rhs) const if( OP == "~" ) { assert(rhs.toString().length > 0); return opBinary!"~"(Path(rhs)); } - void opOpAssign(string OP)(string rhs) if( OP == "~" ) { assert(rhs.length > 0); opOpAssign!"~"(Path(rhs)); } - void opOpAssign(string OP)(PathEntry rhs) if( OP == "~" ) { assert(rhs.toString().length > 0); opOpAssign!"~"(Path(rhs)); } - void opOpAssign(string OP)(Path rhs) if( OP == "~" ) { auto p = this ~ rhs; m_nodes = p.m_nodes; m_endsWithSlash = rhs.m_endsWithSlash; } - - /// Tests two paths for equality using '=='. - bool opEquals(ref const Path rhs) const { - if( m_absolute != rhs.m_absolute ) return false; - if( m_endsWithSlash != rhs.m_endsWithSlash ) return false; - if( m_nodes.length != rhs.length ) return false; - foreach( i; 0 .. m_nodes.length ) - if( m_nodes[i] != rhs.m_nodes[i] ) - return false; - return true; - } - /// ditto - bool opEquals(const Path other) const { return opEquals(other); } - - int opCmp(ref const Path rhs) const { - if( m_absolute != rhs.m_absolute ) return cast(int)m_absolute - cast(int)rhs.m_absolute; - foreach( i; 0 .. min(m_nodes.length, rhs.m_nodes.length) ) - if( m_nodes[i] != rhs.m_nodes[i] ) - return m_nodes[i].opCmp(rhs.m_nodes[i]); - if( m_nodes.length > rhs.m_nodes.length ) return 1; - if( m_nodes.length < rhs.m_nodes.length ) return -1; - return 0; - } -} - -struct PathEntry { - private { - string m_name; - } - - this(string str) - { - assert(str.countUntil('/') < 0 && str.countUntil('\\') < 0); - m_name = str; - } - - string toString() const { return m_name; } - - Path opBinary(string OP)(PathEntry rhs) const if( OP == "~" ) { return Path(cast(immutable)[this, rhs], false); } - - bool opEquals(ref const PathEntry rhs) const { return m_name == rhs.m_name; } - bool opEquals(PathEntry rhs) const { return m_name == rhs.m_name; } - bool opEquals(string rhs) const { return m_name == rhs; } - int opCmp(ref const PathEntry rhs) const { return m_name.cmp(rhs.m_name); } - int opCmp(string rhs) const { return m_name.cmp(rhs); } -} - -private bool isValidFilename(string str) -{ - foreach( ch; str ) - if( ch == '/' || /*ch == ':' ||*/ ch == '\\' ) return false; - return true; -} - -/// Joins two path strings. subpath must be relative. -string joinPath(string basepath, string subpath) -{ - Path p1 = Path(basepath); - Path p2 = Path(subpath); - return (p1 ~ p2).toString(); -} - -/// Splits up a path string into its elements/folders -PathEntry[] splitPath(string path) -{ - if( path.startsWith("/") || path.startsWith("\\") ) path = path[1 .. $]; - if( path.empty ) return null; - if( path.endsWith("/") || path.endsWith("\\") ) path = path[0 .. $-1]; - - // count the number of path nodes - size_t nelements = 0; - foreach( i, char ch; path ) - if( ch == '\\' || ch == '/' ) - nelements++; - nelements++; - - // reserve space for the elements - auto elements = new PathEntry[nelements]; - - // read and return the elements - size_t startidx = 0; - size_t eidx = 0; - foreach( i, char ch; path ) - if( ch == '\\' || ch == '/' ){ - enforce(i - startidx > 0, "Empty path entries not allowed."); - elements[eidx++] = PathEntry(path[startidx .. i]); - startidx = i+1; - } - elements[eidx++] = PathEntry(path[startidx .. $]); - enforce(path.length - startidx > 0, "Empty path entries not allowed."); - assert(eidx == nelements); - return elements; -} diff --git a/source/vibecompat/inet/url.d b/source/vibecompat/inet/url.d deleted file mode 100644 index 66955c6..0000000 --- a/source/vibecompat/inet/url.d +++ /dev/null @@ -1,277 +0,0 @@ -/** - URL parsing routines. - - Copyright: © 2012 RejectedSoftware e.K. - License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. - Authors: Sönke Ludwig -*/ -module vibecompat.inet.url; - -public import vibecompat.inet.path; - -import std.algorithm; -import std.array; -import std.conv; -import std.exception; -import std.string; -import std.uri; - - -/** - Represents a URL decomposed into its components. -*/ -struct Url { - private { - string m_schema; - string m_pathString; - Path m_path; - string m_host; - ushort m_port; - string m_username; - string m_password; - string m_queryString; - string m_anchor; - } - - /// Constructs a new URL object from its components. - this(string schema, string host, ushort port, Path path) - { - m_schema = schema; - m_host = host; - m_port = port; - m_path = path; - m_pathString = path.toString(); - } - /// ditto - this(string schema, Path path) - { - this(schema, null, 0, path); - } - - /** Constructs a URL from its string representation. - - TODO: additional validation required (e.g. valid host and user names and port) - */ - this(string url_string) - { - auto str = url_string; - enforce(str.length > 0, "Empty URL."); - if( str[0] != '/' ){ - auto idx = str.countUntil(':'); - enforce(idx > 0, "No schema in URL:"~str); - m_schema = str[0 .. idx]; - str = str[idx+1 .. $]; - bool requires_host = false; - - switch(m_schema){ - case "http": - case "https": - case "ftp": - case "spdy": - case "sftp": - case "file": - // proto://server/path style - enforce(str.startsWith("//"), "URL must start with proto://..."); - requires_host = true; - str = str[2 .. $]; - goto default; - default: - auto si = str.countUntil('/'); - if( si < 0 ) si = str.length; - auto ai = str[0 .. si].countUntil('@'); - sizediff_t hs = 0; - if( ai >= 0 ){ - hs = ai+1; - auto ci = str[0 .. ai].countUntil(':'); - if( ci >= 0 ){ - m_username = str[0 .. ci]; - m_password = str[ci+1 .. ai]; - } else m_username = str[0 .. ai]; - enforce(m_username.length > 0, "Empty user name in URL."); - } - - m_host = str[hs .. si]; - auto pi = m_host.countUntil(':'); - if(pi > 0) { - enforce(pi < m_host.length-1, "Empty port in URL."); - m_port = to!ushort(m_host[pi+1..$]); - m_host = m_host[0 .. pi]; - } - - enforce(!requires_host || m_schema == "file" || m_host.length > 0, - "Empty server name in URL."); - str = str[si .. $]; - } - } - - this.localURI = str; - } - /// ditto - static Url parse(string url_string) - { - return Url(url_string); - } - - /// The schema/protocol part of the URL - @property string schema() const { return m_schema; } - /// ditto - @property void schema(string v) { m_schema = v; } - - /// The path part of the URL in the original string form - @property string pathString() const { return m_pathString; } - - /// The path part of the URL - @property Path path() const { return m_path; } - /// ditto - @property void path(Path p) - { - m_path = p; - auto pstr = p.toString(); - m_pathString = pstr; - } - - /// The host part of the URL (depends on the schema) - @property string host() const { return m_host; } - /// ditto - @property void host(string v) { m_host = v; } - - /// The port part of the URL (optional) - @property ushort port() const { return m_port; } - /// ditto - @property port(ushort v) { m_port = v; } - - /// The user name part of the URL (optional) - @property string username() const { return m_username; } - /// ditto - @property void username(string v) { m_username = v; } - - /// The password part of the URL (optional) - @property string password() const { return m_password; } - /// ditto - @property void password(string v) { m_password = v; } - - /// The query string part of the URL (optional) - @property string queryString() const { return m_queryString; } - /// ditto - @property void queryString(string v) { m_queryString = v; } - - /// The anchor part of the URL (optional) - @property string anchor() const { return m_anchor; } - - /// The path part plus query string and anchor - @property string localURI() - const { - auto str = appender!string(); - str.reserve(m_pathString.length + 2 + queryString.length + anchor.length); - str.put(encode(path.toString())); - if( queryString.length ) { - str.put("?"); - str.put(queryString); - } - if( anchor.length ) { - str.put("#"); - str.put(anchor); - } - return str.data; - } - /// ditto - @property void localURI(string str) - { - auto ai = str.countUntil('#'); - if( ai >= 0 ){ - m_anchor = str[ai+1 .. $]; - str = str[0 .. ai]; - } - - auto qi = str.countUntil('?'); - if( qi >= 0 ){ - m_queryString = str[qi+1 .. $]; - str = str[0 .. qi]; - } - - m_pathString = str; - m_path = Path(decode(str)); - } - - /// The URL to the parent path with query string and anchor stripped. - @property Url parentUrl() const { - Url ret; - ret.schema = schema; - ret.host = host; - ret.port = port; - ret.username = username; - ret.password = password; - ret.path = path.parentPath; - return ret; - } - - /// Converts this URL object to its string representation. - string toString() - const { - import std.format; - auto dst = appender!string(); - dst.put(schema); - dst.put(":"); - switch(schema){ - default: break; - case "file": - case "http": - case "https": - case "ftp": - case "spdy": - case "sftp": - dst.put("//"); - break; - } - dst.put(host); - if( m_port > 0 ) formattedWrite(dst, ":%d", m_port); - dst.put(localURI); - return dst.data; - } - - bool startsWith(const Url rhs) const { - if( m_schema != rhs.m_schema ) return false; - if( m_host != rhs.m_host ) return false; - // FIXME: also consider user, port, querystring, anchor etc - return path.startsWith(rhs.m_path); - } - - Url opBinary(string OP)(Path rhs) const if( OP == "~" ) { return Url(m_schema, m_host, m_port, m_path ~ rhs); } - Url opBinary(string OP)(PathEntry rhs) const if( OP == "~" ) { return Url(m_schema, m_host, m_port, m_path ~ rhs); } - void opOpAssign(string OP)(Path rhs) if( OP == "~" ) { m_path ~= rhs; } - void opOpAssign(string OP)(PathEntry rhs) if( OP == "~" ) { m_path ~= rhs; } - - /// Tests two URLs for equality using '=='. - bool opEquals(ref const Url rhs) const { - if( m_schema != rhs.m_schema ) return false; - if( m_host != rhs.m_host ) return false; - if( m_path != rhs.m_path ) return false; - return true; - } - /// ditto - bool opEquals(const Url other) const { return opEquals(other); } - - int opCmp(ref const Url rhs) const { - if( m_schema != rhs.m_schema ) return m_schema.cmp(rhs.m_schema); - if( m_host != rhs.m_host ) return m_host.cmp(rhs.m_host); - if( m_path != rhs.m_path ) return m_path.opCmp(rhs.m_path); - return true; - } -} - -unittest { - auto url = Url.parse("https://www.example.net/index.html"); - assert(url.schema == "https", url.schema); - assert(url.host == "www.example.net", url.host); - assert(url.path == Path("/index.html"), url.path.toString()); - - url = Url.parse("http://jo.doe:password@sub.www.example.net:4711/sub2/index.html?query#anchor"); - assert(url.schema == "http", url.schema); - assert(url.username == "jo.doe", url.username); - assert(url.password == "password", url.password); - assert(url.port == 4711, to!string(url.port)); - assert(url.host == "sub.www.example.net", url.host); - assert(url.path.toString() == "/sub2/index.html", url.path.toString()); - assert(url.queryString == "query", url.queryString); - assert(url.anchor == "anchor", url.anchor); -} diff --git a/source/vibecompat/inet/urltransfer.d b/source/vibecompat/inet/urltransfer.d deleted file mode 100644 index 0daca29..0000000 --- a/source/vibecompat/inet/urltransfer.d +++ /dev/null @@ -1,37 +0,0 @@ -/** - Downloading and uploading of data from/to URLs. - - Copyright: © 2012 RejectedSoftware e.K. - License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. - Authors: Sönke Ludwig -*/ -module vibecompat.inet.urltransfer; - -import vibecompat.core.log; -import vibecompat.core.file; -import vibecompat.inet.url; - -import std.exception; -import std.net.curl; -import std.string; - - -/** - Downloads a file from the specified URL. - - Any redirects will be followed until the actual file resource is reached or if the redirection - limit of 10 is reached. Note that only HTTP(S) is currently supported. -*/ -void download(string url, string filename) -{ - auto conn = HTTP(); - static if( is(typeof(&conn.verifyPeer)) ) - conn.verifyPeer = false; - std.net.curl.download(url, filename, conn); -} - -/// ditto -void download(Url url, Path filename) -{ - download(url.toString(), filename.toNativeString()); -}