Enable basic tubes/process test on Windows (#2616)

* Enable basic tubes/process test on Windows

Many tests are Linux specific using qemu or /proc etc. So only test the basic process interaction for now.

* Cleanup docs without using ellipsis for Windows output

* Set newline to b'\r\n' for local Windows processes

But only change it automatically if `context.newline` wasn't set manually.

* Disable process.terminate() test on Windows

Windows doesn't support sending SIGTERM to other processes.

https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal

* libcdb CI cache: add another dns server

[emerg] 1#1: host not found in upstream "debuginfod.elfutils.org" in /etc/nginx/nginx.conf:26

The cache should still start despite not being able to resolve some domain.

Author: peace-maker

docs/source/tubes/processes.rst

@@ -2,10 +2,6 @@
 
    from pwn import *
 
-   # TODO: Remove global POSIX flag
-   import doctest
-   doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX']
-
 :mod:`pwnlib.tubes.process` --- Processes
 ===========================================================
 

docs/source/tubes/serial.rst

@@ -2,10 +2,6 @@
 
    from pwn import *
 
-   # TODO: Remove global POSIX flag
-   import doctest
-   doctest_additional_flags = doctest.OPTIONFLAGS_BY_NAME['POSIX']
-
 :mod:`pwnlib.tubes.serialtube` --- Serial Ports
 ===========================================================
 

pwnlib/tubes/process.py

@@ -133,19 +133,22 @@ class process(tube):
         True
         >>> p.connected('send')
         False
-        >>> p.recvline()
-        b'Hello world\n'
+        >>> p.recvline(drop=True)
+        b'Hello world'
         >>> p.recvuntil(b',')
         b'Wow,'
         >>> p.recvregex(b'.*data')
         b' such data'
-        >>> p.recv()
-        b'\n'
+        >>> p.recv()[-1:] == b'\n'
+        True
         >>> p.recv() # doctest: +ELLIPSIS
         Traceback (most recent call last):
         ...
         EOFError
 
+    .. doctest::
+        :options: +POSIX
+
         >>> p = process('cat')
         >>> d = open('/dev/urandom', 'rb').read(4096)
         >>> p.recv(timeout=0.1)
@@ -289,6 +292,10 @@ def __init__(self, argv = None,
             self.suid = self.uid = None
             self.sgid = self.gid = None
             internal_preexec_fn = None
+            # Expect Windows to use CRLF newlines, but if the user explicitly
+            # set it to something else, don't mess with it.
+            if 'newline' not in context._tls and context.newline == b'\n':
+                self.newline = b'\r\n'
         else:
             # Avoid the need to have to deal with the STDOUT magic value.
             if stderr is STDOUT:
@@ -524,6 +531,9 @@ def program(self):
 
         Example:
 
+        .. doctest::
+            :options: +POSIX +TODO
+
             >>> p = process('/bin/true')
             >>> p.executable == '/bin/true'
             True
@@ -539,6 +549,9 @@ def cwd(self):
 
         Example:
 
+        .. doctest::
+            :options: +POSIX +TODO
+
             >>> p = process('sh')
             >>> p.sendline(b'cd /tmp; echo AAA')
             >>> _ = p.recvuntil(b'AAA')
@@ -687,8 +700,14 @@ def terminate(self):
 
         The process can choose to ignore this signal, so proper cleanup
         is only done in :meth:`kill`/:meth:`close`.
+
+        Note: On Windows, there is no SIGTERM signal, so the Win32 API function ``TerminateProcess()`` is called
+        instead, leading to :meth:`terminate()` and :meth:`kill()` being effectively the same.
         
         Examples:
+
+        .. doctest::
+            :options: +POSIX
         
             >>> p = process(['python', '-u', '-c', 'import signal;signal.signal(signal.SIGTERM, lambda signum,frame: (print("sigterm"),exit(0)));print("ready");import time;time.sleep(10)'])
             >>> p.recvline_contains(b'ready')
@@ -945,7 +964,10 @@ def maps(self):
             read, write, execute, private, shared, string
 
         Example:
-      
+
+        .. doctest::
+            :options: +POSIX +TODO
+
             >>> p = process(['cat'])
             >>> p.sendline(b"meow")
             >>> p.recvline()
@@ -1028,7 +1050,10 @@ def get_mapping(self, path_value, single=True):
         path_value.
 
         Example:
-            
+
+        .. doctest::
+            :options: +POSIX
+
             >>> p = process(['cat'])
             >>> mapping = p.get_mapping('[stack]')
             >>> mapping.path == '[stack]'
@@ -1071,6 +1096,9 @@ def stack_mapping(self, single=True):
 
         Example:
 
+        .. doctest::
+            :options: +POSIX
+
             >>> p = process(['cat'])
             >>> mapping = p.stack_mapping()
             >>> mapping.path
@@ -1100,6 +1128,9 @@ def heap_mapping(self, single=True):
 
         Example:
 
+        .. doctest::
+            :options: +POSIX
+
             >>> p = process(['cat'])
             >>> p.sendline(b'meow')
             >>> p.recvline()
@@ -1132,6 +1163,9 @@ def vdso_mapping(self, single=True):
 
         Example:
 
+        .. doctest::
+            :options: +LINUX
+
             >>> p = process(['cat'])
             >>> mapping = p.vdso_mapping()
             >>> mapping.path
@@ -1161,6 +1195,9 @@ def vvar_mapping(self, single=True):
 
         Example:
 
+        .. doctest::
+            :options: +LINUX
+
             >>> p = process(['cat'])
             >>> mapping = p.vvar_mapping()
             >>> mapping.path
@@ -1191,6 +1228,9 @@ def libc_mapping(self, single=True):
 
         Example:
 
+        .. doctest::
+            :options: +POSIX
+
             >>> p = process(['cat'])
             >>> p.sendline(b'meow')
             >>> p.recvline()
@@ -1271,6 +1311,9 @@ def elf_mapping(self, single=True):
 
         Example:
 
+        .. doctest::
+            :options: +POSIX
+
             >>> p = process(['cat'])
             >>> p.sendline(b'meow')
             >>> p.recvline()
@@ -1309,7 +1352,9 @@ def lib_size(self, path_value):
 
         Example:
 
-            >>> from pwn import *
+        .. doctest::
+            :options: +POSIX
+
             >>> p = process(['cat'])
             >>> p.send(b'meow')
             >>> p.recvuntil(b'meow')
@@ -1349,6 +1394,9 @@ def address_mapping(self, address):
 
         Example:
 
+        .. doctest::
+            :options: +POSIX
+
             >>> p = process(['cat'])
             >>> p.sendline(b'meow')
             >>> p.recvline()
@@ -1403,16 +1451,19 @@ def libc(self):
 
         Example:
 
-        >>> p = process("/bin/cat")
-        >>> p.send(b"meow")
-        >>> p.recvuntil(b"meow")
-        b'meow'
-        >>> libc = p.libc
-        >>> libc is not None
-        True
-        >>> libc # doctest: +SKIP
-        ELF('/lib64/libc-...so')
-        >>> p.close()
+        .. doctest::
+            :options: +POSIX
+
+            >>> p = process("/bin/cat")
+            >>> p.send(b"meow")
+            >>> p.recvuntil(b"meow")
+            b'meow'
+            >>> libc = p.libc
+            >>> libc is not None
+            True
+            >>> libc # doctest: +SKIP
+            ELF('/lib64/libc-...so')
+            >>> p.close()
         """
         from pwnlib.elf import ELF
 
@@ -1490,6 +1541,9 @@ def leak(self, address, count=1):
 
         Example:
 
+        .. doctest::
+            :options: +POSIX +TODO
+
             >>> e = ELF(which('bash-static'))
             >>> p = process(e.path)
 
@@ -1526,6 +1580,9 @@ def writemem(self, address, data):
         
             Let's write data to  the beginning of the mapped memory of the  ELF.
 
+        .. doctest::
+            :options: +POSIX +TODO
+
             >>> context.clear(arch='i386')
             >>> address = 0x100000
             >>> data = cyclic(32)

pwnlib/util/misc.py

@@ -161,8 +161,10 @@ def which(name, all = False, path=None):
 
     Example:
 
-        >>> which('sh') # doctest: +ELLIPSIS +POSIX +TODO
+        >>> which('sh') # doctest: +ELLIPSIS +POSIX
         '.../bin/sh'
+        >>> which('cmd') # doctest: +ELLIPSIS +WINDOWS
+        '...\\cmd.EXE'
     """
     # If name is a path, do not attempt to resolve it.
     if os.path.sep in name:

travis/libcdb_nginx_cache.conf

@@ -10,7 +10,7 @@ http {
     access_log /dev/stdout cache_st;
     # Use Docker's embedded DNS server to resolve hostnames at runtime
     # instead of at startup.
-    resolver 127.0.0.11;
+    resolver 127.0.0.11 8.8.8.8 ipv6=off;
 
     server {
         listen 3000;