From 67afbaf65311ec5fcd8034ea0e8a59e79df68800 Mon Sep 17 00:00:00 2001 From: Zihang Chen Date: Mon, 10 Mar 2014 17:09:19 +0800 Subject: [PATCH] refactor test case classes and add documentation --- testenv/ChangeLog | 23 ++ testenv/ColourTerm.py | 23 -- testenv/FTPServer.py | 162 --------- testenv/HTTPServer.py | 467 ------------------------- testenv/README | 64 ++-- testenv/Test--https.py | 11 +- testenv/Test--spider-r.py | 9 +- testenv/Test-Content-disposition-2.py | 11 +- testenv/Test-Content-disposition.py | 11 +- testenv/Test-Head.py | 11 +- testenv/Test-O.py | 11 +- testenv/Test-Parallel-Proto.py | 14 +- testenv/Test-Post.py | 9 +- testenv/Test-Proto.py | 15 +- testenv/Test-auth-basic-fail.py | 12 +- testenv/Test-auth-basic.py | 13 +- testenv/Test-auth-both.py | 14 +- testenv/Test-auth-digest.py | 9 +- testenv/Test-auth-no-challenge-url.py | 11 +- testenv/Test-auth-no-challenge.py | 11 +- testenv/Test-auth-retcode.py | 10 +- testenv/Test-auth-with-content-disposition.py | 11 +- testenv/Test-c-full.py | 11 +- testenv/Test-cookie-401.py | 11 +- testenv/Test-cookie-domain-mismatch.py | 11 +- testenv/Test-cookie-expires.py | 13 +- testenv/Test-cookie.py | 9 +- testenv/WgetTest.py | 337 ------------------ testenv/certs/wget-cert.pem | 4 +- testenv/conf/__init__.py | 45 +++ testenv/conf/authentication.py | 9 + testenv/conf/expect_header.py | 10 + testenv/conf/expected_files.py | 47 +++ testenv/conf/expected_ret_code.py | 18 + testenv/conf/files_crawled.py | 21 ++ testenv/conf/hook_sample.py | 17 + testenv/conf/local_files.py | 15 + testenv/conf/reject_header.py | 7 + testenv/conf/response.py | 9 + testenv/conf/rule_sample.py | 11 + testenv/conf/send_header.py | 9 + testenv/conf/server_files.py | 17 + testenv/conf/urls.py | 13 + testenv/conf/wget_commands.py | 12 + testenv/exc/__init__.py | 1 + testenv/exc/test_failed.py | 8 + testenv/misc/__init__.py | 1 + testenv/misc/colour_terminal.py | 39 +++ testenv/misc/constants.py | 5 + testenv/misc/wget_file.py | 18 + testenv/server/__init__.py | 1 + testenv/server/ftp/__init__.py | 1 + testenv/server/ftp/ftp_server.py | 162 +++++++++ testenv/server/http/__init__.py | 1 + testenv/server/http/http_server.py | 471 ++++++++++++++++++++++++++ testenv/test/__init__.py | 1 + testenv/test/base_test.py | 247 ++++++++++++++ testenv/test/http_test.py | 53 +++ 58 files changed, 1473 insertions(+), 1124 deletions(-) delete mode 100644 testenv/ColourTerm.py delete mode 100644 testenv/FTPServer.py delete mode 100644 testenv/HTTPServer.py delete mode 100644 testenv/WgetTest.py create mode 100644 testenv/conf/__init__.py create mode 100644 testenv/conf/authentication.py create mode 100644 testenv/conf/expect_header.py create mode 100644 testenv/conf/expected_files.py create mode 100644 testenv/conf/expected_ret_code.py create mode 100644 testenv/conf/files_crawled.py create mode 100644 testenv/conf/hook_sample.py create mode 100644 testenv/conf/local_files.py create mode 100644 testenv/conf/reject_header.py create mode 100644 testenv/conf/response.py create mode 100644 testenv/conf/rule_sample.py create mode 100644 testenv/conf/send_header.py create mode 100644 testenv/conf/server_files.py create mode 100644 testenv/conf/urls.py create mode 100644 testenv/conf/wget_commands.py create mode 100644 testenv/exc/__init__.py create mode 100644 testenv/exc/test_failed.py create mode 100644 testenv/misc/__init__.py create mode 100644 testenv/misc/colour_terminal.py create mode 100644 testenv/misc/constants.py create mode 100644 testenv/misc/wget_file.py create mode 100644 testenv/server/__init__.py create mode 100644 testenv/server/ftp/__init__.py create mode 100644 testenv/server/ftp/ftp_server.py create mode 100644 testenv/server/http/__init__.py create mode 100644 testenv/server/http/http_server.py create mode 100644 testenv/test/__init__.py create mode 100644 testenv/test/base_test.py create mode 100644 testenv/test/http_test.py diff --git a/testenv/ChangeLog b/testenv/ChangeLog index 85bd0f2..8f46f22 100644 --- a/testenv/ChangeLog +++ b/testenv/ChangeLog @@ -1,3 +1,26 @@ +2014-03-09 Zihang Chen + * HTTPServer.py: Move to server/http/http_server.py + * FTPServer.py: Move to server/ftp/ftp_server.py + * WgetTest.py: Refactor and split into test/base_test.py and + test/http_test.py + (CommonMethods): Various changes are made to this class. Briefly, + refactor to BaseTest. And the rule classes and methods are moved to + conf/ in separated files. Documentation is added too. + (HTTPTest): Move to test/http_test.py. + (WgetFile): Move to misc/wget_file.py. + (TestFailed): Move to exc/test_failed.py. + * ColourTerm.py: Move to misc/colour_term.py. Add new functions + print_blue, print_yellow, print_purple, print_red, print_green for + simpler invocation. + * README: Add documentation about how to add custom rules and hooks. + * conf: (new package) Various hooks and rules are put here. + * exc: (new package) Various exceptions are put here. + * misc: (new package) Various constants, helper module or functions + are put here. + * server: (new package) Server modules are put here. + * test: (new package) Test case classes are put here. + + 2014-01-02 Darshit Shah * Makefile.am: Add new Test--https.py to list of tests and EXTRA_DIST. Also replace all tabs with spaces in file for conformity. diff --git a/testenv/ColourTerm.py b/testenv/ColourTerm.py deleted file mode 100644 index d8f6769..0000000 --- a/testenv/ColourTerm.py +++ /dev/null @@ -1,23 +0,0 @@ -import platform -from os import getenv - -T_COLORS = { - 'PURPLE' : '\033[95m', - 'BLUE' : '\033[94m', - 'GREEN' : '\033[92m', - 'YELLOW' : '\033[93m', - 'RED' : '\033[91m', - 'ENDC' : '\033[0m' -} - -def printer (color, string): - if platform.system () == 'Linux': - if getenv ("MAKE_CHECK", "False") == "True": - print (string) - else: - print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC')) - - else: - print (string) - -# vim: set ts=8 sw=3 tw=0 et : diff --git a/testenv/FTPServer.py b/testenv/FTPServer.py deleted file mode 100644 index f7d7771..0000000 --- a/testenv/FTPServer.py +++ /dev/null @@ -1,162 +0,0 @@ -import os -import re -import threading -import socket -import pyftpdlib.__main__ -from pyftpdlib.ioloop import IOLoop -import pyftpdlib.handlers as Handle -from pyftpdlib.servers import FTPServer -from pyftpdlib.authorizers import DummyAuthorizer -from pyftpdlib._compat import PY3, u, b, getcwdu, callable - -class FTPDHandler (Handle.FTPHandler): - - def ftp_LIST (self, path): - try: - iterator = self.run_as_current_user(self.fs.get_list_dir, path) - except (OSError, FilesystemError): - err = sys.exc_info()[1] - why = _strerror (err) - self.respond ('550 %s. ' % why) - else: - if self.isRule ("Bad List") is True: - iter_list = list () - for flist in iterator: - line = re.compile (r'(\s+)').split (flist.decode ('utf-8')) - line[8] = '0' - iter_l = ''.join (line).encode ('utf-8') - iter_list.append (iter_l) - iterator = (n for n in iter_list) - producer = Handle.BufferedIteratorProducer (iterator) - self.push_dtp_data (producer, isproducer=True, cmd="LIST") - return path - - def ftp_PASV (self, line): - if self._epsvall: - self.respond ("501 PASV not allowed after EPSV ALL.") - return - self._make_epasv(extmode=False) - if self.isRule ("FailPASV") is True: - del self.server.global_rules["FailPASV"] - self.socket.close () - - def isRule (self, rule): - rule_obj = self.server.global_rules[rule] - return False if not rule_obj else rule_obj[0] - -class FTPDServer (FTPServer): - - def set_global_rules (self, rules): - self.global_rules = rules - -class FTPd(threading.Thread): - """A threaded FTP server used for running tests. - - This is basically a modified version of the FTPServer class which - wraps the polling loop into a thread. - - The instance returned can be used to start(), stop() and - eventually re-start() the server. - """ - handler = FTPDHandler - server_class = FTPDServer - - def __init__(self, addr=None): - os.mkdir ('server') - os.chdir ('server') - try: - HOST = socket.gethostbyname ('localhost') - except socket.error: - HOST = 'localhost' - USER = 'user' - PASSWD = '12345' - HOME = getcwdu () - - threading.Thread.__init__(self) - self.__serving = False - self.__stopped = False - self.__lock = threading.Lock() - self.__flag = threading.Event() - if addr is None: - addr = (HOST, 0) - - authorizer = DummyAuthorizer() - authorizer.add_user(USER, PASSWD, HOME, perm='elradfmwM') # full perms - authorizer.add_anonymous(HOME) - self.handler.authorizer = authorizer - # lowering buffer sizes = more cycles to transfer data - # = less false positive test failures - self.handler.dtp_handler.ac_in_buffer_size = 32768 - self.handler.dtp_handler.ac_out_buffer_size = 32768 - self.server = self.server_class(addr, self.handler) - self.host, self.port = self.server.socket.getsockname()[:2] - os.chdir ('..') - - def set_global_rules (self, rules): - self.server.set_global_rules (rules) - - def __repr__(self): - status = [self.__class__.__module__ + "." + self.__class__.__name__] - if self.__serving: - status.append('active') - else: - status.append('inactive') - status.append('%s:%s' % self.server.socket.getsockname()[:2]) - return '<%s at %#x>' % (' '.join(status), id(self)) - - @property - def running(self): - return self.__serving - - def start(self, timeout=0.001): - """Start serving until an explicit stop() request. - Polls for shutdown every 'timeout' seconds. - """ - if self.__serving: - raise RuntimeError("Server already started") - if self.__stopped: - # ensure the server can be started again - FTPd.__init__(self, self.server.socket.getsockname(), self.handler) - self.__timeout = timeout - threading.Thread.start(self) - self.__flag.wait() - - def run(self): - self.__serving = True - self.__flag.set() - while self.__serving: - self.__lock.acquire() - self.server.serve_forever(timeout=self.__timeout, blocking=False) - self.__lock.release() - self.server.close_all() - - def stop(self): - """Stop serving (also disconnecting all currently connected - clients) by telling the serve_forever() loop to stop and - waits until it does. - """ - if not self.__serving: - raise RuntimeError("Server not started yet") - self.__serving = False - self.__stopped = True - self.join() - - -def mk_file_sys (file_list): - os.chdir ('server') - for name, content in file_list.items (): - file_h = open (name, 'w') - file_h.write (content) - file_h.close () - os.chdir ('..') - -def filesys (): - fileSys = dict () - os.chdir ('server') - for parent, dirs, files in os.walk ('.'): - for filename in files: - file_handle = open (filename, 'r') - file_content = file_handle.read () - fileSys[filename] = file_content - os.chdir ('..') - return fileSys diff --git a/testenv/HTTPServer.py b/testenv/HTTPServer.py deleted file mode 100644 index e554a10..0000000 --- a/testenv/HTTPServer.py +++ /dev/null @@ -1,467 +0,0 @@ -from http.server import HTTPServer, BaseHTTPRequestHandler -from socketserver import BaseServer -from posixpath import basename, splitext -from base64 import b64encode -from random import random -from hashlib import md5 -import threading -import socket -import re -import ssl -import os - - -class InvalidRangeHeader (Exception): - - """ Create an Exception for handling of invalid Range Headers. """ - # TODO: Eliminate this exception and use only ServerError - - def __init__ (self, err_message): - self.err_message = err_message - -class ServerError (Exception): - def __init__ (self, err_message): - self.err_message = err_message - - -class StoppableHTTPServer (HTTPServer): - - request_headers = list () - - """ Define methods for configuring the Server. """ - - def server_conf (self, filelist, conf_dict): - """ Set Server Rules and File System for this instance. """ - self.server_configs = conf_dict - self.fileSys = filelist - - def server_sett (self, settings): - for settings_key in settings: - setattr (self.RequestHandlerClass, settings_key, settings[settings_key]) - - def get_req_headers (self): - return self.request_headers - -class HTTPSServer (StoppableHTTPServer): - - def __init__ (self, address, handler): - BaseServer.__init__ (self, address, handler) - print (os.getcwd()) - CERTFILE = os.path.abspath (os.path.join ('..', 'certs', 'wget-cert.pem')) - print (CERTFILE) - fop = open (CERTFILE) - print (fop.readline()) - self.socket = ssl.wrap_socket ( - sock = socket.socket (self.address_family, self.socket_type), - ssl_version = ssl.PROTOCOL_TLSv1, - certfile = CERTFILE, - server_side = True - ) - self.server_bind () - self.server_activate () - -class WgetHTTPRequestHandler (BaseHTTPRequestHandler): - - """ Define methods for handling Test Checks. """ - - def get_rule_list (self, name): - r_list = self.rules.get (name) if name in self.rules else None - return r_list - - -class _Handler (WgetHTTPRequestHandler): - - """ Define Handler Methods for different Requests. """ - - InvalidRangeHeader = InvalidRangeHeader - protocol_version = 'HTTP/1.1' - - """ Define functions for various HTTP Requests. """ - - def do_HEAD (self): - self.send_head ("HEAD") - - def do_GET (self): - content, start = self.send_head ("GET") - if content: - if start is None: - self.wfile.write (content.encode ('utf-8')) - else: - self.wfile.write (content.encode ('utf-8')[start:]) - - def do_POST (self): - path = self.path[1:] - self.rules = self.server.server_configs.get (path) - if not self.custom_response (): - return (None, None) - if path in self.server.fileSys: - body_data = self.get_body_data () - self.send_response (200) - self.send_header ("Content-type", "text/plain") - content = self.server.fileSys.pop (path) + "\n" + body_data - total_length = len (content) - self.server.fileSys[path] = content - self.send_header ("Content-Length", total_length) - self.finish_headers () - try: - self.wfile.write (content.encode ('utf-8')) - except Exception: - pass - else: - self.send_put (path) - - def do_PUT (self): - path = self.path[1:] - self.rules = self.server.server_configs.get (path) - if not self.custom_response (): - return (None, None) - self.server.fileSys.pop (path, None) - self.send_put (path) - - """ End of HTTP Request Method Handlers. """ - - """ Helper functions for the Handlers. """ - - def parse_range_header (self, header_line, length): - if header_line is None: - return None - if not header_line.startswith ("bytes="): - raise InvalidRangeHeader ("Cannot parse header Range: %s" % - (header_line)) - regex = re.match (r"^bytes=(\d*)\-$", header_line) - range_start = int (regex.group (1)) - if range_start >= length: - raise InvalidRangeHeader ("Range Overflow") - return range_start - - def get_body_data (self): - cLength_header = self.headers.get ("Content-Length") - cLength = int (cLength_header) if cLength_header is not None else 0 - body_data = self.rfile.read (cLength).decode ('utf-8') - return body_data - - def send_put (self, path): - body_data = self.get_body_data () - self.send_response (201) - self.server.fileSys[path] = body_data - self.send_header ("Content-type", "text/plain") - self.send_header ("Content-Length", len (body_data)) - self.finish_headers () - try: - self.wfile.write (body_data.encode ('utf-8')) - except Exception: - pass - - def SendHeader (self, header_obj): - pass -# headers_list = header_obj.headers -# for header_line in headers_list: -# print (header_line + " : " + headers_list[header_line]) -# self.send_header (header_line, headers_list[header_line]) - - def send_cust_headers (self): - header_obj = self.get_rule_list ('SendHeader') - if header_obj: - for header in header_obj.headers: - self.send_header (header, header_obj.headers[header]) - - def finish_headers (self): - self.send_cust_headers () - self.end_headers () - - def Response (self, resp_obj): - self.send_response (resp_obj.response_code) - self.finish_headers () - raise ServerError ("Custom Response code sent.") - - def custom_response (self): - codes = self.get_rule_list ('Response') - if codes: - self.send_response (codes.response_code) - self.finish_headers () - return False - else: - return True - - def base64 (self, data): - string = b64encode (data.encode ('utf-8')) - return string.decode ('utf-8') - - def send_challenge (self, auth_type): - if auth_type == "Both": - self.send_challenge ("Digest") - self.send_challenge ("Basic") - return - if auth_type == "Basic": - challenge_str = 'Basic realm="Wget-Test"' - elif auth_type == "Digest" or auth_type == "Both_inline": - self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest () - self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest () - challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' %( - self.nonce, - self.opaque) - challenge_str += ', qop="auth"' - if auth_type == "Both_inline": - challenge_str = 'Basic realm="Wget-Test", ' + challenge_str - self.send_header ("WWW-Authenticate", challenge_str) - - def authorize_Basic (self, auth_header, auth_rule): - if auth_header is None or auth_header.split(' ')[0] != 'Basic': - return False - else: - self.user = auth_rule.auth_user - self.passw = auth_rule.auth_pass - auth_str = "Basic " + self.base64 (self.user + ":" + self.passw) - return True if auth_str == auth_header else False - - def parse_auth_header (self, auth_header): - n = len("Digest ") - auth_header = auth_header[n:].strip() - items = auth_header.split(", ") - key_values = [i.split("=", 1) for i in items] - key_values = [(k.strip(), v.strip().replace('"', '')) for k, v in key_values] - return dict(key_values) - - def KD (self, secret, data): - return self.H (secret + ":" + data) - - def H (self, data): - return md5 (data.encode ('utf-8')).hexdigest () - - def A1 (self): - return "%s:%s:%s" % (self.user, "Test", self.passw) - - def A2 (self, params): - return "%s:%s" % (self.command, params["uri"]) - - def check_response (self, params): - if "qop" in params: - data_str = params['nonce'] \ - + ":" + params['nc'] \ - + ":" + params['cnonce'] \ - + ":" + params['qop'] \ - + ":" + self.H (self.A2 (params)) - else: - data_str = params['nonce'] + ":" + self.H (self.A2 (params)) - resp = self.KD (self.H (self.A1 ()), data_str) - - return True if resp == params['response'] else False - - def authorize_Digest (self, auth_header, auth_rule): - if auth_header is None or auth_header.split(' ')[0] != 'Digest': - return False - else: - self.user = auth_rule.auth_user - self.passw = auth_rule.auth_pass - params = self.parse_auth_header (auth_header) - pass_auth = True - if self.user != params['username'] or \ - self.nonce != params['nonce'] or self.opaque != params['opaque']: - pass_auth = False - req_attribs = ['username', 'realm', 'nonce', 'uri', 'response'] - for attrib in req_attribs: - if not attrib in params: - pass_auth = False - if not self.check_response (params): - pass_auth = False - return pass_auth - - def authorize_Both (self, auth_header, auth_rule): - return False - - def authorize_Both_inline (self, auth_header, auth_rule): - return False - - def Authentication (self, auth_rule): - try: - self.handle_auth (auth_rule) - except ServerError as se: - self.send_response (401, "Authorization Required") - self.send_challenge (auth_rule.auth_type) - self.finish_headers () - raise ServerError (se.__str__()) - - def handle_auth (self, auth_rule): - is_auth = True - auth_header = self.headers.get ("Authorization") - required_auth = auth_rule.auth_type - if required_auth == "Both" or required_auth == "Both_inline": - auth_type = auth_header.split(' ')[0] if auth_header else required_auth - else: - auth_type = required_auth - try: - assert hasattr (self, "authorize_" + auth_type) - is_auth = getattr (self, "authorize_" + auth_type) (auth_header, auth_rule) - except AssertionError: - raise ServerError ("Authentication Mechanism " + auth_rule + " not supported") - except AttributeError as ae: - raise ServerError (ae.__str__()) - if is_auth is False: - raise ServerError ("Unable to Authenticate") - - def is_authorized (self): - is_auth = True - auth_rule = self.get_rule_list ('Authentication') - if auth_rule: - auth_header = self.headers.get ("Authorization") - req_auth = auth_rule.auth_type - if req_auth == "Both" or req_auth == "Both_inline": - auth_type = auth_header.split(' ')[0] if auth_header else req_auth - else: - auth_type = req_auth - assert hasattr (self, "authorize_" + auth_type) - is_auth = getattr (self, "authorize_" + auth_type) (auth_header, auth_rule) - if is_auth is False: - self.send_response (401) - self.send_challenge (auth_type) - self.finish_headers () - return is_auth - - def ExpectHeader (self, header_obj): - exp_headers = header_obj.headers - for header_line in exp_headers: - header_recd = self.headers.get (header_line) - if header_recd is None or header_recd != exp_headers[header_line]: - self.send_error (400, "Expected Header " + header_line + " not found") - self.finish_headers () - raise ServerError ("Header " + header_line + " not found") - - def expect_headers (self): - """ This is modified code to handle a few changes. Should be removed ASAP """ - exp_headers_obj = self.get_rule_list ('ExpectHeader') - if exp_headers_obj: - exp_headers = exp_headers_obj.headers - for header_line in exp_headers: - header_re = self.headers.get (header_line) - if header_re is None or header_re != exp_headers[header_line]: - self.send_error (400, 'Expected Header not Found') - self.end_headers () - return False - return True - - def RejectHeader (self, header_obj): - rej_headers = header_obj.headers - for header_line in rej_headers: - header_recd = self.headers.get (header_line) - if header_recd is not None and header_recd == rej_headers[header_line]: - self.send_error (400, 'Blackisted Header ' + header_line + ' received') - self.finish_headers () - raise ServerError ("Header " + header_line + ' received') - - def reject_headers (self): - rej_headers = self.get_rule_list ("RejectHeader") - if rej_headers: - rej_headers = rej_headers.headers - for header_line in rej_headers: - header_re = self.headers.get (header_line) - if header_re is not None and header_re == rej_headers[header_line]: - self.send_error (400, 'Blacklisted Header was Sent') - self.end_headers () - return False - return True - - def __log_request (self, method): - req = method + " " + self.path - self.server.request_headers.append (req) - - def send_head (self, method): - """ Common code for GET and HEAD Commands. - This method is overriden to use the fileSys dict. - - The method variable contains whether this was a HEAD or a GET Request. - According to RFC 2616, the server should not differentiate between - the two requests, however, we use it here for a specific test. - """ - - if self.path == "/": - path = "index.html" - else: - path = self.path[1:] - - self.__log_request (method) - - if path in self.server.fileSys: - self.rules = self.server.server_configs.get (path) - - for rule_name in self.rules: - try: - assert hasattr (self, rule_name) - getattr (self, rule_name) (self.rules [rule_name]) - except AssertionError as ae: - msg = "Method " + rule_name + " not defined" - self.send_error (500, msg) - return (None, None) - except ServerError as se: - print (se.__str__()) - return (None, None) - - content = self.server.fileSys.get (path) - content_length = len (content) - try: - self.range_begin = self.parse_range_header ( - self.headers.get ("Range"), content_length) - except InvalidRangeHeader as ae: - # self.log_error("%s", ae.err_message) - if ae.err_message == "Range Overflow": - self.send_response (416) - self.finish_headers () - return (None, None) - else: - self.range_begin = None - if self.range_begin is None: - self.send_response (200) - else: - self.send_response (206) - self.send_header ("Accept-Ranges", "bytes") - self.send_header ("Content-Range", - "bytes %d-%d/%d" % (self.range_begin, - content_length - 1, - content_length)) - content_length -= self.range_begin - cont_type = self.guess_type (path) - self.send_header ("Content-type", cont_type) - self.send_header ("Content-Length", content_length) - self.finish_headers () - return (content, self.range_begin) - else: - self.send_error (404, "Not Found") - return (None, None) - - def guess_type (self, path): - base_name = basename ("/" + path) - name, ext = splitext (base_name) - extension_map = { - ".txt" : "text/plain", - ".css" : "text/css", - ".html" : "text/html" - } - if ext in extension_map: - return extension_map[ext] - else: - return "text/plain" - - -class HTTPd (threading.Thread): - server_class = StoppableHTTPServer - handler = _Handler - def __init__ (self, addr=None): - threading.Thread.__init__ (self) - if addr is None: - addr = ('localhost', 0) - self.server_inst = self.server_class (addr, self.handler) - self.server_address = self.server_inst.socket.getsockname()[:2] - - def run (self): - self.server_inst.serve_forever () - - def server_conf (self, file_list, server_rules): - self.server_inst.server_conf (file_list, server_rules) - - def server_sett (self, settings): - self.server_inst.server_sett (settings) - -class HTTPSd (HTTPd): - - server_class = HTTPSServer - -# vim: set ts=4 sts=4 sw=4 tw=80 et : diff --git a/testenv/README b/testenv/README index 09f226a..1075944 100644 --- a/testenv/README +++ b/testenv/README @@ -10,25 +10,30 @@ Run the './configure' command to generate the Makefile and then run 'make check' to execute the Test Suite. Use the '-j n' option with 'make check' to execute n tests simultaneously. -File List: +Structure: ================================================================================ - * HTTPServer.py: This file contains a custom, programmatically configurable - HTTP Server for testing Wget. It runs an instance of Python's http.server - module. + * server: This package contains custom programmatically configurable servers + (both HTTP and FTP) for testing Wget. The HTTP server runs an instance of + Python's http.server module. The FTP server is to be implemented. - * WgetTest.py: This file contains various functions and global variables for - each instance of the server that is initiated. It includes functions to - start and stop the server, to initialze the test environment and to cleanup - after a test. + * test: This package contains the test case classes for HTTP and FTP. The + test case classes includes methods for initializing and cleaning up of the + test environment. + + * exc: This package contains custom exception classes used in this test + suite. + + * conf: This package contains the configuration classes for servers to be + configured with. + + * misc: This package contains several functions, classes and constants used + in this test suite. * Test-Proto.py: This is a prototype Test Case file. The file defines all the acceptable elements and their uses. Typically, one must copy this file and edit it for writing Test Cases. - * ColourTerm.py: A custom library for printing coloured output to the - terminal. Currently it only supports 4 colours in a *nix environment. - Working: ================================================================================ @@ -101,7 +106,7 @@ Next, is the const variable, TEST_NAME that defines the name of the Test. Each File in the Test must be represented as a WgetFile object. The WgetFile Class has the following prototype: -WgetFile (String name, String contents, String timestamp, dict rules) +WgetFile (str name, str contents, str timestamp, dict rules) None except name is a mandatory paramter, one may pass only those parameters that are required by the File object. @@ -138,10 +143,11 @@ Both, the HTTPTest and FTPTest modules have the same prototype: pre_hook, test_options, post_hook, - servers + protocols } -name expects the string name, and is usually passed the TEST_NAME variable, -the three hooks, expect python dictionary objects and servers is an integer. +name should be a string, and is usually passed to the TEST_NAME variable, +the three hooks should be Python dict objects and protocols should be a list of +protocols, like [HTTP, HTTPS]. Valid File Rules: ================================================================================ @@ -191,6 +197,7 @@ executed. The currently supported options are: * Urls : A list of the filenames that Wget must attempt to download. The complete URL will be created and passed to Wget automatically. + (alias URLs) * WgetCommands : A string consisting of the various commandline switches sent to Wget upon invokation. Any data placed between {{ }} in this string will be replaced with the contents of self. before being passed to @@ -207,7 +214,7 @@ hooks are usually used to run checks on the data, files downloaded, return code, etc. The following hooks are currently supported: * ExpectedRetcode : This is an integer value of the ReturnCode with which - Wget is expected to exit. + Wget is expected to exit. (alias: ExpectedRetCode) * ExpectedFiles : This is a list of WgetFile objects of the files that must exist locally on disk in the Test directory. * FilesCrawled : This requires a list of the Requests that the server is @@ -223,13 +230,26 @@ recommended method for writing new Test Case files is to copy Test-Proto.py and modify it to ones needs. In case you require any functionality that is not currently defined in List of -Rules defined above, you should add the required code in WgetTest.py. -In most cases, one requires a new Rule to be added for the Server to follow. -In such a case, create a new Class in WgetTest.py with the same name as the Rule -and define an __init__ () function to handle the data. A method must also be -defined in HTTPTest / FTPTest modules to handle the said Rule. +Rules defined above, you should implement a new class in the conf package. The +file name doesn't matters (though it's better to give it an appropriate name). +The new rule class should be like this: + +============================================================= +from conf import register + + address@hidden() +class MyNewRule: + + def __init__(self, rule_arg): + self.rule_arg = rule_arg + # your rule initialization code goes here +============================================================= + +Additionally, A method with the exact name of the newly created rule class must + also be defined in HTTPTest / FTPTest modules to handle the said Rule. -Once a new Test File is created, it must be added to the TESTS variable in +Once a new Test File is created, it must be added to the TESTS variable in Makefile.am. This way the Test will be executed on running a 'make check'. If a Test is expected to fail on the current master branch, then the Test should also be added to the XFAIL_TESTS variable. This will allow expected failures to diff --git a/testenv/Test--https.py b/testenv/Test--https.py index 17252b6..7bd4a0a 100755 --- a/testenv/Test--https.py +++ b/testenv/Test--https.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile, HTTPS, HTTP +from misc.wget_file import WgetFile +from test.http_test import HTTPTest, HTTPS + + +# This test ensures that Wget can download files from HTTPS Servers. -""" - This test ensures that Wget can download files from HTTPS Servers -""" TEST_NAME = "HTTPS Downloads" ############# File Definitions ############################################### File1 = "Would you like some Tea?" @@ -45,7 +46,7 @@ err = HTTPTest ( pre_hook=pre_test, test_params=test_options, post_hook=post_test, - servers=Servers + protocols=Servers ).begin () exit (err) diff --git a/testenv/Test--spider-r.py b/testenv/Test--spider-r.py index b770a9f..0663193 100755 --- a/testenv/Test--spider-r.py +++ b/testenv/Test--spider-r.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from misc.wget_file import WgetFile +from test.http_test import HTTPTest + + +# This test executed Wget in Spider mode with recursive retrieval. -""" - This test executed Wget in Spider mode with recursive retrieval. -""" TEST_NAME = "Recursive Spider" ############# File Definitions ############################################### mainpage = """ diff --git a/testenv/Test-Content-disposition-2.py b/testenv/Test-Content-disposition-2.py index c2512e1..dbd156e 100755 --- a/testenv/Test-Content-disposition-2.py +++ b/testenv/Test-Content-disposition-2.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures that Wget parses the Content-Disposition header +# correctly and creates the appropriate file when the said filename exists. -""" - This test ensures that Wget parses the Content-Disposition header - correctly and creates the appropriate file when the said filename exists. -""" TEST_NAME = "Content Disposition Clobber" ############# File Definitions ############################################### File1 = "Teapot" diff --git a/testenv/Test-Content-disposition.py b/testenv/Test-Content-disposition.py index 0a81dea..966e374 100755 --- a/testenv/Test-Content-disposition.py +++ b/testenv/Test-Content-disposition.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures that Wget parses the Content-Disposition header +# correctly and creates a local file accordingly. -""" - This test ensures that Wget parses the Content-Disposition header - correctly and creates a local file accordingly. -""" TEST_NAME = "Content Disposition Header" ############# File Definitions ############################################### File1 = """All that is gold does not glitter, diff --git a/testenv/Test-Head.py b/testenv/Test-Head.py index 49aaa41..0dae0bb 100755 --- a/testenv/Test-Head.py +++ b/testenv/Test-Head.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures that Wget correctly handles responses to HEAD requests +# and does not actually download any data. -""" - This test ensures that Wget correctly handles responses to HEAD requests - and does not actually download any data -""" TEST_NAME = "HEAD Requests" ############# File Definitions ############################################### File1 = "You shall not pass!" diff --git a/testenv/Test-O.py b/testenv/Test-O.py index 613fbcd..b62a58e 100755 --- a/testenv/Test-O.py +++ b/testenv/Test-O.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures that Wget correctly handles the -O command for output +# file names. -""" - This test ensures that Wget correctly handles the -O command for output - filenames. -""" TEST_NAME = "Output Filename Command" ############# File Definitions ############################################### File1 = "Test Contents." diff --git a/testenv/Test-Parallel-Proto.py b/testenv/Test-Parallel-Proto.py index e7aae2e..8c45f4c 100755 --- a/testenv/Test-Parallel-Proto.py +++ b/testenv/Test-Parallel-Proto.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile, HTTP, HTTPS +from misc.constants import HTTP +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This is a Prototype Test File for multiple servers. +# Ideally this File should be copied and edited to write new tests. -""" - This is a Prototype Test File for multiple servers. - Ideally this File should be copied and edited to write new tests. -""" TEST_NAME = "Parallel Prototype" ############# File Definitions ############################################### File1 = "Would you like some Tea?" @@ -46,7 +48,7 @@ err = HTTPTest ( pre_hook=pre_test, test_params=test_options, post_hook=post_test, - servers=Servers + protocols=Servers ).begin () exit (err) diff --git a/testenv/Test-Post.py b/testenv/Test-Post.py index 632326f..908d491 100755 --- a/testenv/Test-Post.py +++ b/testenv/Test-Post.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# Simple test for HTTP POST Requests using the --method command. -""" - Simple test for HTTP POST Requests usiong the --method command -""" TEST_NAME = "HTTP POST Requests" ############# File Definitions ############################################### File1 = """A reader lives a thousand lives before he dies, said Jojen. diff --git a/testenv/Test-Proto.py b/testenv/Test-Proto.py index eaafdc1..25ffeb8 100755 --- a/testenv/Test-Proto.py +++ b/testenv/Test-Proto.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile, HTTP, HTTPS +from misc.constants import HTTP +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + + +# This is a Prototype Test File. +# Ideally this File should be copied and edited to write new tests. -""" - This is a Prototype Test File. - Ideally this File should be copied and edited to write new tests. -""" TEST_NAME = "Prototype" ############# File Definitions ############################################### File1 = "Would you like some Tea?" @@ -67,7 +70,7 @@ err = HTTPTest ( pre_hook=pre_test, test_params=test_options, post_hook=post_test, - server=Servers + protocols=Servers ).begin () exit (err) diff --git a/testenv/Test-auth-basic-fail.py b/testenv/Test-auth-basic-fail.py index 894e96d..2ac55d7 100755 --- a/testenv/Test-auth-basic-fail.py +++ b/testenv/Test-auth-basic-fail.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + + +# This test ensures that Wget returns the correct exit code when Basic +# authentication fails due to a username/password error. -""" - This test ensures that Wget returns the correct exit code when Basic - authentcation failes due to a username/password error. -""" TEST_NAME = "Basic Authentication Failure" ############# File Definitions ############################################### File1 = "I am an invisble man." diff --git a/testenv/Test-auth-basic.py b/testenv/Test-auth-basic.py index 96141e0..1584650 100755 --- a/testenv/Test-auth-basic.py +++ b/testenv/Test-auth-basic.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures Wget's Basic Authorization Negotiation. +# Also, we ensure that Wget saves the host after a successfull auth and +# doesn't wait for a challenge the second time. -""" - This test ensures Wget's Basic Authorization Negotiation. - Also, we ensure that Wget saves the host after a successfull auth and - doesn't wait for a challenge the second time. -""" TEST_NAME = "Basic Authorization" ############# File Definitions ############################################### File1 = "I am an invisble man." diff --git a/testenv/Test-auth-both.py b/testenv/Test-auth-both.py index 9837134..67131bf 100755 --- a/testenv/Test-auth-both.py +++ b/testenv/Test-auth-both.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + + +# This test ensures Wget's Basic Authorization Negotiation. +# Also, we ensure that Wget saves the host after a successful auth and +# doesn't wait for a challenge the second time. -""" - This test ensures Wget's Basic Authorization Negotiation. - Also, we ensure that Wget saves the host after a successfull auth and - doesn't wait for a challenge the second time. -""" TEST_NAME = "Multiple authentication support" ############# File Definitions ############################################### File1 = "Would you like some Tea?" diff --git a/testenv/Test-auth-digest.py b/testenv/Test-auth-digest.py index a66b2c9..b54258b 100755 --- a/testenv/Test-auth-digest.py +++ b/testenv/Test-auth-digest.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures Wget's Digest Authorization Negotiation. -""" - This test ensures Wget's Digest Authorization Negotiation. -""" TEST_NAME = "Digest Authorization" ############# File Definitions ############################################### File1 = "Need a cookie?" diff --git a/testenv/Test-auth-no-challenge-url.py b/testenv/Test-auth-no-challenge-url.py index eb88ac5..2628caa 100755 --- a/testenv/Test-auth-no-challenge-url.py +++ b/testenv/Test-auth-no-challenge-url.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures Wget's Basic Authorization Negotiation, when credentials +# are provided in URL. -""" - This test ensures Wget's Basic Authorization Negotiation, when credentials - are provided in-URL -""" TEST_NAME = "Auth no challenge in URL" ############# File Definitions ############################################### File1 = "Need a cookie?" diff --git a/testenv/Test-auth-no-challenge.py b/testenv/Test-auth-no-challenge.py index 774bd59..b9349e3 100755 --- a/testenv/Test-auth-no-challenge.py +++ b/testenv/Test-auth-no-challenge.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures Wget's Basic Authorization Negotiation, when the +# --auth-no-challenge command is used. -""" - This test ensures Wget's Basic Authorization Negotiation, when the - --auth-no-challenge command is used. -""" TEST_NAME = "Auth No Challenge Command" ############# File Definitions ############################################### File1 = "Need a cookie?" diff --git a/testenv/Test-auth-retcode.py b/testenv/Test-auth-retcode.py index adbcbb0..096a59e 100755 --- a/testenv/Test-auth-retcode.py +++ b/testenv/Test-auth-retcode.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile -""" - This test ensures that Wget returns the correct return code when sent - a 403 Forbidden by the Server. -""" + +# This test ensures that Wget returns the correct return code when sent +# a 403 Forbidden by the Server. TEST_NAME = "Forbidden Retcode" diff --git a/testenv/Test-auth-with-content-disposition.py b/testenv/Test-auth-with-content-disposition.py index 50e08ba..efdb197 100755 --- a/testenv/Test-auth-with-content-disposition.py +++ b/testenv/Test-auth-with-content-disposition.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures that Wget handles Content-Disposition correctly when +# coupled with Authentication. -""" - This test ensures that Wget handles Content-Disposition correctly when - coupled with Authentication -""" TEST_NAME = "Authentication with Content Disposition" ############# File Definitions ############################################### File1 = "Need a cookie?" diff --git a/testenv/Test-c-full.py b/testenv/Test-c-full.py index 87ffc52..029322c 100755 --- a/testenv/Test-c-full.py +++ b/testenv/Test-c-full.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# Test Wget's response when the file requested already exists on disk with +# a file size greater than or equal to the requested file. -""" - Test Wget's response when the file requested already exists on disk with - a filesize greater than or equal to the requested file. -""" TEST_NAME = "Test continue option" ############# File Definitions ############################################### File1 = "abababababababababababababababababababababababababababababababababab" diff --git a/testenv/Test-cookie-401.py b/testenv/Test-cookie-401.py index 9ca9641..8a23606 100755 --- a/testenv/Test-cookie-401.py +++ b/testenv/Test-cookie-401.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures that Wget stores the cookie even in the event of a +# 401 Unauthorized Response. -""" - This test ensures that Wget stores the cookie even in the event of a - 401 Unauthorized Response -""" TEST_NAME = "Basic Cookie 401 Response" ############# File Definitions ############################################### File1 = """All happy families are alike; diff --git a/testenv/Test-cookie-domain-mismatch.py b/testenv/Test-cookie-domain-mismatch.py index ae108d9..68f44c3 100755 --- a/testenv/Test-cookie-domain-mismatch.py +++ b/testenv/Test-cookie-domain-mismatch.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures that Wget identifies bad servers trying to set cookies +# for a different domain and rejects them. -""" - This test ensures that Wget identifies bad servers trying to set cookies - for a different domain and rejects them. -""" TEST_NAME = "Cookie Domain Mismatch" ############# File Definitions ############################################### File1 = "Would you care for a cup of coffee?" diff --git a/testenv/Test-cookie-expires.py b/testenv/Test-cookie-expires.py index 2ae3bf9..5889d2c 100755 --- a/testenv/Test-cookie-expires.py +++ b/testenv/Test-cookie-expires.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures that Wget handles Cookie expiry dates correctly. +# Simultaneously, we also check if multiple cookies to the same domain +# are handled correctly. -""" - This test ensures that Wget handles Cookie expiry dates correctly. - Simultaneuously, we also check if multiple cookies to the same domain - are handled correctly -""" TEST_NAME = "Cookie Expires" ############# File Definitions ############################################### File1 = "Hello World!" diff --git a/testenv/Test-cookie.py b/testenv/Test-cookie.py index 7f5b093..bb8a104 100755 --- a/testenv/Test-cookie.py +++ b/testenv/Test-cookie.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 from sys import exit -from WgetTest import HTTPTest, WgetFile +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + + +# This test ensures that Wget's cookie jar support works correctly. -""" - This test ensures that Wget's cookie jar support works correctly. -""" TEST_NAME = "Basic Cookie Functionality" ############# File Definitions ############################################### File1 = """All happy families are alike; diff --git a/testenv/WgetTest.py b/testenv/WgetTest.py deleted file mode 100644 index 6470d93..0000000 --- a/testenv/WgetTest.py +++ /dev/null @@ -1,337 +0,0 @@ -import os -import shutil -import shlex -import sys -import traceback -import HTTPServer -import re -import time -from subprocess import call -from ColourTerm import printer -from difflib import unified_diff - -HTTP = "HTTP" -HTTPS = "HTTPS" - -""" A Custom Exception raised by the Test Environment. """ - -class TestFailed (Exception): - - def __init__ (self, error): - self.error = error - - -""" Class that defines methods common to both HTTP and FTP Tests. """ - -class CommonMethods: - TestFailed = TestFailed - - def init_test_env (self, name): - testDir = name + "-test" - try: - os.mkdir (testDir) - except FileExistsError: - shutil.rmtree (testDir) - os.mkdir (testDir) - os.chdir (testDir) - self.tests_passed = True - - def get_domain_addr (self, addr): - self.port = str (addr[1]) - return addr[0] + ":" + str(addr[1]) + "/" - - def exec_wget (self, options, urls, domain_list): - cmd_line = self.get_cmd_line (options, urls, domain_list) - params = shlex.split (cmd_line) - print (params) - if os.getenv ("SERVER_WAIT"): - time.sleep (float (os.getenv ("SERVER_WAIT"))) - try: - retcode = call (params) - except FileNotFoundError as filenotfound: - raise TestFailed ( - "The Wget Executable does not exist at the expected path") - return retcode - - def get_cmd_line (self, options, urls, domain_list): - TEST_PATH = os.path.abspath (".") - WGET_PATH = os.path.join (TEST_PATH, "..", "..", "src", "wget") - WGET_PATH = os.path.abspath (WGET_PATH) - cmd_line = WGET_PATH + " " + options + " " - for i in range (0, self.servers): - for url in urls[i]: - protocol = "http://" if self.server_types[i] is "HTTP" else "https://" - cmd_line += protocol + domain_list[i] + url + " " -# for url in urls: -# cmd_line += domain_list[0] + url + " " - print (cmd_line) - return cmd_line - - def __test_cleanup (self): - testDir = self.name + "-test" - os.chdir ('..') - try: - if os.getenv ("NO_CLEANUP") is None: - shutil.rmtree (testDir) - except Exception as ae: - print ("Unknown Exception while trying to remove Test Environment.") - - def _exit_test (self): - self.__test_cleanup () - - def begin (self): - return 0 if self.tests_passed else 100 - - """ Methods to check if the Test Case passes or not. """ - - def __gen_local_filesys (self): - file_sys = dict () - for parent, dirs, files in os.walk ('.'): - for name in files: - onefile = dict () - # Create the full path to file, removing the leading ./ - # Might not work on non-unix systems. Someone please test. - filepath = os.path.join (parent, name) - file_handle = open (filepath, 'r') - file_content = file_handle.read () - onefile['content'] = file_content - filepath = filepath[2:] - file_sys[filepath] = onefile - file_handle.close () - return file_sys - - - def __check_downloaded_files (self, exp_filesys): - local_filesys = self.__gen_local_filesys () - for files in exp_filesys: - if files.name in local_filesys: - local_file = local_filesys.pop (files.name) - if files.content != local_file ['content']: - for line in unified_diff (local_file['content'], files.content, fromfile="Actual", tofile="Expected"): - sys.stderr.write (line) - raise TestFailed ("Contents of " + files.name + " do not match") - else: - raise TestFailed ("Expected file " + files.name + " not found") - if local_filesys: - print (local_filesys) - raise TestFailed ("Extra files downloaded.") - - def _replace_substring (self, string): - pattern = re.compile ('\{\{\w+\}\}') - match_obj = pattern.search (string) - if match_obj is not None: - rep = match_obj.group() - temp = getattr (self, rep.strip ('{}')) - string = string.replace (rep, temp) - return string - - - """ Test Rule Definitions """ - """ This should really be taken out soon. All this extra stuff to ensure - re-use of old code is crap. Someone needs to re-write it. The new rework - branch is much better written, but integrating it requires effort. - All these classes should never exist. The whole server needs to modified. - """ - - class Authentication: - def __init__ (self, auth_obj): - self.auth_type = auth_obj['Type'] - self.auth_user = auth_obj['User'] - self.auth_pass = auth_obj['Pass'] - - class ExpectHeader: - def __init__ (self, header_obj): - self.headers = header_obj - - class RejectHeader: - def __init__ (self, header_obj): - self.headers = header_obj - - class Response: - def __init__ (self, retcode): - self.response_code = retcode - - class SendHeader: - def __init__ (self, header_obj): - self.headers = header_obj - - def get_server_rules (self, file_obj): - """ The handling of expect header could be made much better when the - options are parsed in a true and better fashion. For an example, - see the commented portion in Test-basic-auth.py. - """ - server_rules = dict () - for rule in file_obj.rules: - r_obj = getattr (self, rule) (file_obj.rules[rule]) - server_rules[rule] = r_obj - return server_rules - - """ Pre-Test Hook Function Calls """ - - def ServerFiles (self, server_files): - for i in range (0, self.servers): - file_list = dict () - server_rules = dict () - for file_obj in server_files[i]: - content = self._replace_substring (file_obj.content) - file_list[file_obj.name] = content - rule_obj = self.get_server_rules (file_obj) - server_rules[file_obj.name] = rule_obj - self.server_list[i].server_conf (file_list, server_rules) - - def LocalFiles (self, local_files): - for file_obj in local_files: - file_handler = open (file_obj.name, "w") - file_handler.write (file_obj.content) - file_handler.close () - - def ServerConf (self, server_settings): - for i in range (0, self.servers): - self.server_list[i].server_sett (server_settings) - - """ Test Option Function Calls """ - - def WgetCommands (self, command_list): - self.options = self._replace_substring (command_list) - - def Urls (self, url_list): - self.urls = url_list - - """ Post-Test Hook Function Calls """ - - def ExpectedRetcode (self, retcode): - if self.act_retcode != retcode: - pr = "Return codes do not match.\nExpected: " + str(retcode) + "\nActual: " + str(self.act_retcode) - raise TestFailed (pr) - - def ExpectedFiles (self, exp_filesys): - self.__check_downloaded_files (exp_filesys) - - def FilesCrawled (self, Request_Headers): - for i in range (0, self.servers): - headers = set(Request_Headers[i]) - o_headers = self.Request_remaining[i] - header_diff = headers.symmetric_difference (o_headers) - if len(header_diff) is not 0: - printer ("RED", str (header_diff)) - raise TestFailed ("Not all files were crawled correctly") - - -""" Class for HTTP Tests. """ - -class HTTPTest (CommonMethods): - -# Temp Notes: It is expected that when pre-hook functions are executed, only an empty test-dir exists. -# pre-hook functions are executed just prior to the call to Wget is made. -# post-hook functions will be executed immediately after the call to Wget returns. - - def __init__ ( - self, - name="Unnamed Test", - pre_hook=dict(), - test_params=dict(), - post_hook=dict(), - servers=[HTTP] - ): - try: - self.Server_setup (name, pre_hook, test_params, post_hook, servers) - except TestFailed as tf: - printer ("RED", "Error: " + tf.error) - self.tests_passed = False - except Exception as ae: - printer ("RED", "Unhandled Exception Caught.") - print ( ae.__str__ ()) - traceback.print_exc () - self.tests_passed = False - else: - printer ("GREEN", "Test Passed") - finally: - self._exit_test () - - def Server_setup (self, name, pre_hook, test_params, post_hook, servers): - self.name = name - self.server_types = servers - self.servers = len (servers) - printer ("BLUE", "Running Test " + self.name) - self.init_test_env (name) - self.server_list = list() - self.domain_list = list() - for server_type in servers: - server_inst = getattr (self, "init_" + server_type + "_Server") () - self.server_list.append (server_inst) - domain = self.get_domain_addr (server_inst.server_address) - self.domain_list.append (domain) - #self.server = self.init_HTTP_Server () - #self.domain = self.get_domain_addr (self.server.server_address) - - self.pre_hook_call (pre_hook) - self.call_test (test_params) - self.post_hook_call (post_hook) - - def pre_hook_call (self, pre_hook): - for pre_hook_func in pre_hook: - try: - assert hasattr (self, pre_hook_func) - except AssertionError as ae: - self.stop_HTTP_Server () - raise TestFailed ("Pre Test Function " + pre_hook_func + " not defined.") - getattr (self, pre_hook_func) (pre_hook[pre_hook_func]) - - def call_test (self, test_params): - for test_func in test_params: - try: - assert hasattr (self, test_func) - except AssertionError as ae: - self.stop_HTTP_Server () - raise TestFailed ("Test Option " + test_func + " unknown.") - getattr (self, test_func) (test_params[test_func]) - - try: - self.act_retcode = self.exec_wget (self.options, self.urls, self.domain_list) - except TestFailed as tf: - self.stop_HTTP_Server () - raise TestFailed (tf.__str__ ()) - self.stop_HTTP_Server () - - def post_hook_call (self, post_hook): - for post_hook_func in post_hook: - try: - assert hasattr (self, post_hook_func) - except AssertionError as ae: - raise TestFailed ("Post Test Function " + post_hook_func + " not defined.") - getattr (self, post_hook_func) (post_hook[post_hook_func]) - - def init_HTTP_Server (self): - server = HTTPServer.HTTPd () - server.start () - return server - - def init_HTTPS_Server (self): - server = HTTPServer.HTTPSd () - server.start () - return server - - def stop_HTTP_Server (self): - self.Request_remaining = list () - for server in self.server_list: - server_req = server.server_inst.get_req_headers () - self.Request_remaining.append (server_req) - server.server_inst.shutdown () - -""" WgetFile is a File Data Container object """ - -class WgetFile: - - def __init__ ( - self, - name, - content="Test Contents", - timestamp=None, - rules=dict() - ): - self.name = name - self.content = content - self.timestamp = timestamp - self.rules = rules - -# vim: set ts=4 sts=4 sw=4 tw=80 et : diff --git a/testenv/certs/wget-cert.pem b/testenv/certs/wget-cert.pem index b83069e..bd48eb8 100644 --- a/testenv/certs/wget-cert.pem +++ b/testenv/certs/wget-cert.pem @@ -1,5 +1,5 @@ -----BEGIN PRIVATE KEY----- -MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBAMV8qEpuSVUdWaAY +IICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBAMV8qEpuSVUdWaAY F2N1ljGEJ/907Og5B0aZLeDskmLAOohKMWTiiSx+lseXVD/Zf/LaFfy/+q0Rk5+o pFEPEEjadvdxogb9HPwjfj48ng74yV1c5ZGRx/aIeIJN9cacfs4J5NlT3ZPiV8/2 mpBurBYvta5tneUl+lx4NHTEBmjTAgMBAAECgYBHlFlDMRovWYYEuvavPA2GQQpm @@ -15,7 +15,7 @@ Csz0tHWWF7z6V38TmExac6Ef07clFQtlHoooAH1t2D8l2g205hlJAkBfeghbZDdY kjrIQqDV8MLu -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICODCCAaGgAwIBAgIJAOiSkPuPcAwqMA0GCSqGSIb3DQEBBQUAMDUxCzAJBgNV +IICODCCAaGgAwIBAgIJAOiSkPuPcAwqMA0GCSqGSIb3DQEBBQUAMDUxCzAJBgNV BAYTAklOMRMwEQYDVQQIDApTb21lLVN0YXRlMREwDwYDVQQKDAhHTlUgV2dldDAe Fw0xMzEyMDcwNTA3NTRaFw0xNDEyMDcwNTA3NTRaMDUxCzAJBgNVBAYTAklOMRMw EQYDVQQIDApTb21lLVN0YXRlMREwDwYDVQQKDAhHTlUgV2dldDCBnzANBgkqhkiG diff --git a/testenv/conf/__init__.py b/testenv/conf/__init__.py new file mode 100644 index 0000000..96611dc --- /dev/null +++ b/testenv/conf/__init__.py @@ -0,0 +1,45 @@ +import os + +# this file implements the mechanism of conf class auto-registration, +# don't modify this file if you have no idea what you're doing + +def gen_hook(): + hook_table = {} + + class Wrapper: + """ + Decorator class which implements the conf class registration. + """ + def __init__(self, alias=None): + self.alias = alias + + + def __call__(self, cls): + # register the class object with the name of the class + hook_table[cls.__name__] = cls + if self.alias: + # also register the alias of the class + hook_table[self.alias] = cls + + return cls + + def find_hook(name): + if name in hook_table: + return hook_table[name] + else: + raise AttributeError + + return Wrapper, find_hook + +register, find_conf = gen_hook() + +for module in os.listdir(os.path.dirname(__file__)): + # import every module under this package except __init__.py, + # so that the decorator `register` applies + # (nothing happens if the script is not loaded) + if module != '__init__.py' and module.endswith('.py'): + module_name = module[:-3] + mod = __import__('%s.%s' % (__name__, module_name), + globals(), + locals()) + diff --git a/testenv/conf/authentication.py b/testenv/conf/authentication.py new file mode 100644 index 0000000..1b73b50 --- /dev/null +++ b/testenv/conf/authentication.py @@ -0,0 +1,9 @@ +from conf import register + + address@hidden() +class Authentication: + def __init__ (self, auth_obj): + self.auth_type = auth_obj['Type'] + self.auth_user = auth_obj['User'] + self.auth_pass = auth_obj['Pass'] diff --git a/testenv/conf/expect_header.py b/testenv/conf/expect_header.py new file mode 100644 index 0000000..2f372d8 --- /dev/null +++ b/testenv/conf/expect_header.py @@ -0,0 +1,10 @@ +from conf import register + + address@hidden() +class ExpectHeader: + def __init__(self, header_obj): + self.headers = header_obj + + + diff --git a/testenv/conf/expected_files.py b/testenv/conf/expected_files.py new file mode 100644 index 0000000..09144d8 --- /dev/null +++ b/testenv/conf/expected_files.py @@ -0,0 +1,47 @@ +from difflib import unified_diff +import os +import sys +from conf import register +from exc.test_failed import TestFailed + + address@hidden() +class ExpectedFiles: + def __init__(self, expected_fs): + self.expected_fs = expected_fs + + + @staticmethod + def gen_local_fs_snapshot(): + snapshot = {} + for parent, dirs, files in os.walk('.'): + for name in files: + f = {'content': ''} + file_path = os.path.join(parent, name) + with open(file_path) as fp: + f['content'] = fp.read() + snapshot[file_path[2:]] = f + + return snapshot + + + def __call__(self, test_obj): + local_fs = self.gen_local_fs_snapshot() + for file in self.expected_fs: + if file.name in local_fs: + local_file = local_fs.pop(file.name) + if file.content != local_file['content']: + for line in unified_diff(local_file['content'], + file.content, + fromfile='Actual', + tofile='Expected'): + print(line, file=sys.stderr) + raise TestFailed('Contents of %s do not match.' % file.name) + else: + raise TestFailed('Expected file %s not found.' % file.name) + if local_fs: + print(local_fs) + raise TestFailed('Extra files downloaded.') + + + diff --git a/testenv/conf/expected_ret_code.py b/testenv/conf/expected_ret_code.py new file mode 100644 index 0000000..3666787 --- /dev/null +++ b/testenv/conf/expected_ret_code.py @@ -0,0 +1,18 @@ +from exc.test_failed import TestFailed +from conf import register + + address@hidden(alias='ExpectedRetcode') +class ExpectedRetCode: + def __init__(self, expected_ret_code): + self.expected_ret_code = expected_ret_code + + + def __call__(self, test_obj): + if test_obj.ret_code != self.expected_ret_code: + failure = "Return codes do not match.\n" \ + "Expected: %s\n" \ + "Actual: %s" % (self.expected_ret_code, + test_obj.ret_code) + raise TestFailed(failure) + diff --git a/testenv/conf/files_crawled.py b/testenv/conf/files_crawled.py new file mode 100644 index 0000000..ffc3c6e --- /dev/null +++ b/testenv/conf/files_crawled.py @@ -0,0 +1,21 @@ +from misc.colour_terminal import print_red +from conf import register +from exc.test_failed import TestFailed + + address@hidden() +class FilesCrawled: + def __init__(self, request_headers): + self.request_headers = request_headers + + + def __call__(self, test_obj): + for headers, remaining in zip(map(set, self.request_headers), + test_obj.request_remaining()): + diff = headers.symmetric_difference(remaining) + + if diff: + print_red(diff) + raise TestFailed('Not all files were crawled correctly.') + + diff --git a/testenv/conf/hook_sample.py b/testenv/conf/hook_sample.py new file mode 100644 index 0000000..58db304 --- /dev/null +++ b/testenv/conf/hook_sample.py @@ -0,0 +1,17 @@ +from exc.test_failed import TestFailed +from conf import register + +# this file is a hook example + address@hidden(alias='SampleAlias') +class SampleHook: + def __init__(self, sample_hook_arg): + # do conf initialization here + self.arg = sample_hook_arg + + + def __call__(self, test_obj): + # implement hook here + # if you need the test case instance, refer to test_obj + pass + diff --git a/testenv/conf/local_files.py b/testenv/conf/local_files.py new file mode 100644 index 0000000..0442f35 --- /dev/null +++ b/testenv/conf/local_files.py @@ -0,0 +1,15 @@ +from conf import register + + address@hidden() +class LocalFiles: + def __init__(self, local_files): + self.local_files = local_files + + + def __call__(self, _): + for f in self.local_files: + with open(f.name, 'w') as fp: + fp.write(f.content) + + diff --git a/testenv/conf/reject_header.py b/testenv/conf/reject_header.py new file mode 100644 index 0000000..81c76c4 --- /dev/null +++ b/testenv/conf/reject_header.py @@ -0,0 +1,7 @@ +from conf import register + + address@hidden() +class RejectHeader: + def __init__ (self, header_obj): + self.headers = header_obj diff --git a/testenv/conf/response.py b/testenv/conf/response.py new file mode 100644 index 0000000..67502d3 --- /dev/null +++ b/testenv/conf/response.py @@ -0,0 +1,9 @@ +from conf import register + + address@hidden() +class Response: + def __init__(self, ret_code): + self.response_code = ret_code + + diff --git a/testenv/conf/rule_sample.py b/testenv/conf/rule_sample.py new file mode 100644 index 0000000..8052c70 --- /dev/null +++ b/testenv/conf/rule_sample.py @@ -0,0 +1,11 @@ +from conf import register + + address@hidden(alias='SampleRuleAlias') +class SampleRule: + def __init__(self, rule): + # do rule initialization here + # you may also need to implement a method the same name of this + # class in server/protocol/protocol_server.py to apply this rule. + self.rule = rule + diff --git a/testenv/conf/send_header.py b/testenv/conf/send_header.py new file mode 100644 index 0000000..00b2e30 --- /dev/null +++ b/testenv/conf/send_header.py @@ -0,0 +1,9 @@ +from conf import register + + address@hidden() +class SendHeader: + def __init__(self, header_obj): + self.headers = header_obj + + diff --git a/testenv/conf/server_files.py b/testenv/conf/server_files.py new file mode 100644 index 0000000..81c575a --- /dev/null +++ b/testenv/conf/server_files.py @@ -0,0 +1,17 @@ +from conf import register + + address@hidden() +class ServerFiles: + def __init__(self, server_files): + self.server_files = server_files + + + def __call__(self, test_obj): + for server, files in zip(test_obj.servers, self.server_files): + rules = {f.name: test_obj.get_server_rules(f) + for f in files} + files = {f.name: test_obj.replace_quotes(f.content) + for f in files} + server.config(files, rules) + diff --git a/testenv/conf/urls.py b/testenv/conf/urls.py new file mode 100644 index 0000000..00e7fff --- /dev/null +++ b/testenv/conf/urls.py @@ -0,0 +1,13 @@ +from conf import register + + address@hidden(alias='Urls') +class URLs: + def __init__(self, urls): + self.urls = urls + + + def __call__(self, test_obj): + test_obj.urls = self.urls + + diff --git a/testenv/conf/wget_commands.py b/testenv/conf/wget_commands.py new file mode 100644 index 0000000..8365e47 --- /dev/null +++ b/testenv/conf/wget_commands.py @@ -0,0 +1,12 @@ +from conf import register + + address@hidden() +class WgetCommands: + def __init__(self, commands): + self.wget_options = commands + + + def __call__(self, test_obj): + test_obj.wget_options = test_obj.replace_quotes(self.wget_options) + diff --git a/testenv/exc/__init__.py b/testenv/exc/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/testenv/exc/__init__.py @@ -0,0 +1 @@ + diff --git a/testenv/exc/test_failed.py b/testenv/exc/test_failed.py new file mode 100644 index 0000000..9942a1b --- /dev/null +++ b/testenv/exc/test_failed.py @@ -0,0 +1,8 @@ + + +class TestFailed (Exception): + + """ A Custom Exception raised by the Test Environment. """ + + def __init__ (self, error): + self.error = error \ No newline at end of file diff --git a/testenv/misc/__init__.py b/testenv/misc/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/testenv/misc/__init__.py @@ -0,0 +1 @@ + diff --git a/testenv/misc/colour_terminal.py b/testenv/misc/colour_terminal.py new file mode 100644 index 0000000..f2cddae --- /dev/null +++ b/testenv/misc/colour_terminal.py @@ -0,0 +1,39 @@ +from functools import partial +import platform +from os import getenv +import sys + +T_COLORS = { + 'PURPLE' : '\033[95m', + 'BLUE' : '\033[94m', + 'GREEN' : '\033[92m', + 'YELLOW' : '\033[93m', + 'RED' : '\033[91m', + 'ENDC' : '\033[0m' +} + +def printer (color, string): + if platform.system () == 'Linux': + if getenv ("MAKE_CHECK", "False") == "True": + print (string) + else: + print (T_COLORS.get (color) + string + T_COLORS.get ('ENDC')) + + else: + print (string) + + +# Because of grammar checks of pycharm, printers are hard-coded. +# for color in T_COLORS: +# setattr(sys.modules[__name__], +# 'print_%s' % color.lower(), +# partial(printer, color)) + +print_blue = partial(printer, 'BLUE') +print_red = partial(printer, 'RED') +print_green = partial(printer, 'GREEN') +print_purple = partial(printer, 'PURPLE') +print_yellow = partial(printer, 'YELLOW') + + +# vim: set ts=8 sw=3 tw=0 et : diff --git a/testenv/misc/constants.py b/testenv/misc/constants.py new file mode 100644 index 0000000..1ac74a8 --- /dev/null +++ b/testenv/misc/constants.py @@ -0,0 +1,5 @@ + + + +HTTP = "HTTP" +HTTPS = "HTTPS" \ No newline at end of file diff --git a/testenv/misc/wget_file.py b/testenv/misc/wget_file.py new file mode 100644 index 0000000..2814533 --- /dev/null +++ b/testenv/misc/wget_file.py @@ -0,0 +1,18 @@ + + +class WgetFile: + + """ WgetFile is a File Data Container object """ + + def __init__ ( + self, + name, + content="Test Contents", + timestamp=None, + rules=None + ): + self.name = name + self.content = content + self.timestamp = timestamp + self.rules = rules or {} + diff --git a/testenv/server/__init__.py b/testenv/server/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/testenv/server/__init__.py @@ -0,0 +1 @@ + diff --git a/testenv/server/ftp/__init__.py b/testenv/server/ftp/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/testenv/server/ftp/__init__.py @@ -0,0 +1 @@ + diff --git a/testenv/server/ftp/ftp_server.py b/testenv/server/ftp/ftp_server.py new file mode 100644 index 0000000..f7d7771 --- /dev/null +++ b/testenv/server/ftp/ftp_server.py @@ -0,0 +1,162 @@ +import os +import re +import threading +import socket +import pyftpdlib.__main__ +from pyftpdlib.ioloop import IOLoop +import pyftpdlib.handlers as Handle +from pyftpdlib.servers import FTPServer +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib._compat import PY3, u, b, getcwdu, callable + +class FTPDHandler (Handle.FTPHandler): + + def ftp_LIST (self, path): + try: + iterator = self.run_as_current_user(self.fs.get_list_dir, path) + except (OSError, FilesystemError): + err = sys.exc_info()[1] + why = _strerror (err) + self.respond ('550 %s. ' % why) + else: + if self.isRule ("Bad List") is True: + iter_list = list () + for flist in iterator: + line = re.compile (r'(\s+)').split (flist.decode ('utf-8')) + line[8] = '0' + iter_l = ''.join (line).encode ('utf-8') + iter_list.append (iter_l) + iterator = (n for n in iter_list) + producer = Handle.BufferedIteratorProducer (iterator) + self.push_dtp_data (producer, isproducer=True, cmd="LIST") + return path + + def ftp_PASV (self, line): + if self._epsvall: + self.respond ("501 PASV not allowed after EPSV ALL.") + return + self._make_epasv(extmode=False) + if self.isRule ("FailPASV") is True: + del self.server.global_rules["FailPASV"] + self.socket.close () + + def isRule (self, rule): + rule_obj = self.server.global_rules[rule] + return False if not rule_obj else rule_obj[0] + +class FTPDServer (FTPServer): + + def set_global_rules (self, rules): + self.global_rules = rules + +class FTPd(threading.Thread): + """A threaded FTP server used for running tests. + + This is basically a modified version of the FTPServer class which + wraps the polling loop into a thread. + + The instance returned can be used to start(), stop() and + eventually re-start() the server. + """ + handler = FTPDHandler + server_class = FTPDServer + + def __init__(self, addr=None): + os.mkdir ('server') + os.chdir ('server') + try: + HOST = socket.gethostbyname ('localhost') + except socket.error: + HOST = 'localhost' + USER = 'user' + PASSWD = '12345' + HOME = getcwdu () + + threading.Thread.__init__(self) + self.__serving = False + self.__stopped = False + self.__lock = threading.Lock() + self.__flag = threading.Event() + if addr is None: + addr = (HOST, 0) + + authorizer = DummyAuthorizer() + authorizer.add_user(USER, PASSWD, HOME, perm='elradfmwM') # full perms + authorizer.add_anonymous(HOME) + self.handler.authorizer = authorizer + # lowering buffer sizes = more cycles to transfer data + # = less false positive test failures + self.handler.dtp_handler.ac_in_buffer_size = 32768 + self.handler.dtp_handler.ac_out_buffer_size = 32768 + self.server = self.server_class(addr, self.handler) + self.host, self.port = self.server.socket.getsockname()[:2] + os.chdir ('..') + + def set_global_rules (self, rules): + self.server.set_global_rules (rules) + + def __repr__(self): + status = [self.__class__.__module__ + "." + self.__class__.__name__] + if self.__serving: + status.append('active') + else: + status.append('inactive') + status.append('%s:%s' % self.server.socket.getsockname()[:2]) + return '<%s at %#x>' % (' '.join(status), id(self)) + + @property + def running(self): + return self.__serving + + def start(self, timeout=0.001): + """Start serving until an explicit stop() request. + Polls for shutdown every 'timeout' seconds. + """ + if self.__serving: + raise RuntimeError("Server already started") + if self.__stopped: + # ensure the server can be started again + FTPd.__init__(self, self.server.socket.getsockname(), self.handler) + self.__timeout = timeout + threading.Thread.start(self) + self.__flag.wait() + + def run(self): + self.__serving = True + self.__flag.set() + while self.__serving: + self.__lock.acquire() + self.server.serve_forever(timeout=self.__timeout, blocking=False) + self.__lock.release() + self.server.close_all() + + def stop(self): + """Stop serving (also disconnecting all currently connected + clients) by telling the serve_forever() loop to stop and + waits until it does. + """ + if not self.__serving: + raise RuntimeError("Server not started yet") + self.__serving = False + self.__stopped = True + self.join() + + +def mk_file_sys (file_list): + os.chdir ('server') + for name, content in file_list.items (): + file_h = open (name, 'w') + file_h.write (content) + file_h.close () + os.chdir ('..') + +def filesys (): + fileSys = dict () + os.chdir ('server') + for parent, dirs, files in os.walk ('.'): + for filename in files: + file_handle = open (filename, 'r') + file_content = file_handle.read () + fileSys[filename] = file_content + os.chdir ('..') + return fileSys diff --git a/testenv/server/http/__init__.py b/testenv/server/http/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/testenv/server/http/__init__.py @@ -0,0 +1 @@ + diff --git a/testenv/server/http/http_server.py b/testenv/server/http/http_server.py new file mode 100644 index 0000000..c287a22 --- /dev/null +++ b/testenv/server/http/http_server.py @@ -0,0 +1,471 @@ +from http.server import HTTPServer, BaseHTTPRequestHandler +from socketserver import BaseServer +from posixpath import basename, splitext +from base64 import b64encode +from random import random +from hashlib import md5 +import threading +import socket +import re +import ssl +import os + + +class InvalidRangeHeader (Exception): + + """ Create an Exception for handling of invalid Range Headers. """ + # TODO: Eliminate this exception and use only ServerError + + def __init__ (self, err_message): + self.err_message = err_message + +class ServerError (Exception): + def __init__ (self, err_message): + self.err_message = err_message + + +class StoppableHTTPServer (HTTPServer): + + request_headers = list () + + """ Define methods for configuring the Server. """ + + def config(self, files, rules): + """ Set Server Rules and File System for this instance. """ + self.server_configs = rules + self.fileSys = files + + def server_sett (self, settings): + for settings_key in settings: + setattr (self.RequestHandlerClass, settings_key, settings[settings_key]) + + def get_req_headers (self): + return self.request_headers + +class HTTPSServer (StoppableHTTPServer): + + def __init__ (self, address, handler): + BaseServer.__init__ (self, address, handler) + print (os.getcwd()) + CERTFILE = os.path.abspath(os.path.join('..', 'certs', 'wget-cert.pem')) + print (CERTFILE) + fop = open (CERTFILE) + print (fop.readline()) + self.socket = ssl.wrap_socket ( + sock = socket.socket (self.address_family, self.socket_type), + ssl_version = ssl.PROTOCOL_TLSv1, + certfile = CERTFILE, + server_side = True + ) + self.server_bind () + self.server_activate () + +class WgetHTTPRequestHandler (BaseHTTPRequestHandler): + + """ Define methods for handling Test Checks. """ + + def get_rule_list (self, name): + r_list = self.rules.get (name) if name in self.rules else None + return r_list + + +class _Handler (WgetHTTPRequestHandler): + + """ Define Handler Methods for different Requests. """ + + InvalidRangeHeader = InvalidRangeHeader + protocol_version = 'HTTP/1.1' + + """ Define functions for various HTTP Requests. """ + + def do_HEAD (self): + self.send_head ("HEAD") + + def do_GET (self): + content, start = self.send_head ("GET") + if content: + if start is None: + self.wfile.write (content.encode ('utf-8')) + else: + self.wfile.write (content.encode ('utf-8')[start:]) + + def do_POST (self): + path = self.path[1:] + self.rules = self.server.server_configs.get (path) + if not self.custom_response (): + return (None, None) + if path in self.server.fileSys: + body_data = self.get_body_data () + self.send_response (200) + self.send_header ("Content-type", "text/plain") + content = self.server.fileSys.pop (path) + "\n" + body_data + total_length = len (content) + self.server.fileSys[path] = content + self.send_header ("Content-Length", total_length) + self.finish_headers () + try: + self.wfile.write (content.encode ('utf-8')) + except Exception: + pass + else: + self.send_put (path) + + def do_PUT (self): + path = self.path[1:] + self.rules = self.server.server_configs.get (path) + if not self.custom_response (): + return (None, None) + self.server.fileSys.pop (path, None) + self.send_put (path) + + ######################################## + # End of HTTP Request Method Handlers. # + ######################################## + + ###################################### + # Helper functions for the Handlers. # + ###################################### + + def parse_range_header (self, header_line, length): + if header_line is None: + return None + if not header_line.startswith ("bytes="): + raise InvalidRangeHeader ("Cannot parse header Range: %s" % + (header_line)) + regex = re.match (r"^bytes=(\d*)\-$", header_line) + range_start = int (regex.group (1)) + if range_start >= length: + raise InvalidRangeHeader ("Range Overflow") + return range_start + + def get_body_data (self): + cLength_header = self.headers.get ("Content-Length") + cLength = int (cLength_header) if cLength_header is not None else 0 + body_data = self.rfile.read (cLength).decode ('utf-8') + return body_data + + def send_put (self, path): + body_data = self.get_body_data () + self.send_response (201) + self.server.fileSys[path] = body_data + self.send_header ("Content-type", "text/plain") + self.send_header ("Content-Length", len (body_data)) + self.finish_headers () + try: + self.wfile.write (body_data.encode ('utf-8')) + except Exception: + pass + + def SendHeader (self, header_obj): + pass +# headers_list = header_obj.headers +# for header_line in headers_list: +# print (header_line + " : " + headers_list[header_line]) +# self.send_header (header_line, headers_list[header_line]) + + def send_cust_headers (self): + header_obj = self.get_rule_list ('SendHeader') + if header_obj: + for header in header_obj.headers: + self.send_header (header, header_obj.headers[header]) + + def finish_headers (self): + self.send_cust_headers () + self.end_headers () + + def Response (self, resp_obj): + self.send_response (resp_obj.response_code) + self.finish_headers () + raise ServerError ("Custom Response code sent.") + + def custom_response (self): + codes = self.get_rule_list ('Response') + if codes: + self.send_response (codes.response_code) + self.finish_headers () + return False + else: + return True + + def base64 (self, data): + string = b64encode (data.encode ('utf-8')) + return string.decode ('utf-8') + + def send_challenge (self, auth_type): + if auth_type == "Both": + self.send_challenge ("Digest") + self.send_challenge ("Basic") + return + if auth_type == "Basic": + challenge_str = 'Basic realm="Wget-Test"' + elif auth_type == "Digest" or auth_type == "Both_inline": + self.nonce = md5 (str (random ()).encode ('utf-8')).hexdigest () + self.opaque = md5 (str (random ()).encode ('utf-8')).hexdigest () + challenge_str = 'Digest realm="Test", nonce="%s", opaque="%s"' %( + self.nonce, + self.opaque) + challenge_str += ', qop="auth"' + if auth_type == "Both_inline": + challenge_str = 'Basic realm="Wget-Test", ' + challenge_str + self.send_header ("WWW-Authenticate", challenge_str) + + def authorize_Basic (self, auth_header, auth_rule): + if auth_header is None or auth_header.split(' ')[0] != 'Basic': + return False + else: + self.user = auth_rule.auth_user + self.passw = auth_rule.auth_pass + auth_str = "Basic " + self.base64 (self.user + ":" + self.passw) + return True if auth_str == auth_header else False + + def parse_auth_header (self, auth_header): + n = len("Digest ") + auth_header = auth_header[n:].strip() + items = auth_header.split(", ") + key_values = [i.split("=", 1) for i in items] + key_values = [(k.strip(), v.strip().replace('"', '')) for k, v in key_values] + return dict(key_values) + + def KD (self, secret, data): + return self.H (secret + ":" + data) + + def H (self, data): + return md5 (data.encode ('utf-8')).hexdigest () + + def A1 (self): + return "%s:%s:%s" % (self.user, "Test", self.passw) + + def A2 (self, params): + return "%s:%s" % (self.command, params["uri"]) + + def check_response (self, params): + if "qop" in params: + data_str = params['nonce'] \ + + ":" + params['nc'] \ + + ":" + params['cnonce'] \ + + ":" + params['qop'] \ + + ":" + self.H (self.A2 (params)) + else: + data_str = params['nonce'] + ":" + self.H (self.A2 (params)) + resp = self.KD (self.H (self.A1 ()), data_str) + + return True if resp == params['response'] else False + + def authorize_Digest (self, auth_header, auth_rule): + if auth_header is None or auth_header.split(' ')[0] != 'Digest': + return False + else: + self.user = auth_rule.auth_user + self.passw = auth_rule.auth_pass + params = self.parse_auth_header (auth_header) + pass_auth = True + if self.user != params['username'] or \ + self.nonce != params['nonce'] or self.opaque != params['opaque']: + pass_auth = False + req_attribs = ['username', 'realm', 'nonce', 'uri', 'response'] + for attrib in req_attribs: + if not attrib in params: + pass_auth = False + if not self.check_response (params): + pass_auth = False + return pass_auth + + def authorize_Both (self, auth_header, auth_rule): + return False + + def authorize_Both_inline (self, auth_header, auth_rule): + return False + + def Authentication (self, auth_rule): + try: + self.handle_auth (auth_rule) + except ServerError as se: + self.send_response (401, "Authorization Required") + self.send_challenge (auth_rule.auth_type) + self.finish_headers () + raise ServerError (se.__str__()) + + def handle_auth (self, auth_rule): + is_auth = True + auth_header = self.headers.get ("Authorization") + required_auth = auth_rule.auth_type + if required_auth == "Both" or required_auth == "Both_inline": + auth_type = auth_header.split(' ')[0] if auth_header else required_auth + else: + auth_type = required_auth + try: + assert hasattr (self, "authorize_" + auth_type) + is_auth = getattr (self, "authorize_" + auth_type) (auth_header, auth_rule) + except AssertionError: + raise ServerError ("Authentication Mechanism " + auth_rule + " not supported") + except AttributeError as ae: + raise ServerError (ae.__str__()) + if is_auth is False: + raise ServerError ("Unable to Authenticate") + + def is_authorized (self): + is_auth = True + auth_rule = self.get_rule_list ('Authentication') + if auth_rule: + auth_header = self.headers.get ("Authorization") + req_auth = auth_rule.auth_type + if req_auth == "Both" or req_auth == "Both_inline": + auth_type = auth_header.split(' ')[0] if auth_header else req_auth + else: + auth_type = req_auth + assert hasattr (self, "authorize_" + auth_type) + is_auth = getattr (self, "authorize_" + auth_type) (auth_header, auth_rule) + if is_auth is False: + self.send_response (401) + self.send_challenge (auth_type) + self.finish_headers () + return is_auth + + def ExpectHeader (self, header_obj): + exp_headers = header_obj.headers + for header_line in exp_headers: + header_recd = self.headers.get (header_line) + if header_recd is None or header_recd != exp_headers[header_line]: + self.send_error (400, "Expected Header " + header_line + " not found") + self.finish_headers () + raise ServerError ("Header " + header_line + " not found") + + def expect_headers (self): + """ This is modified code to handle a few changes. Should be removed ASAP """ + exp_headers_obj = self.get_rule_list ('ExpectHeader') + if exp_headers_obj: + exp_headers = exp_headers_obj.headers + for header_line in exp_headers: + header_re = self.headers.get (header_line) + if header_re is None or header_re != exp_headers[header_line]: + self.send_error (400, 'Expected Header not Found') + self.end_headers () + return False + return True + + def RejectHeader (self, header_obj): + rej_headers = header_obj.headers + for header_line in rej_headers: + header_recd = self.headers.get (header_line) + if header_recd is not None and header_recd == rej_headers[header_line]: + self.send_error (400, 'Blackisted Header ' + header_line + ' received') + self.finish_headers () + raise ServerError ("Header " + header_line + ' received') + + def reject_headers (self): + rej_headers = self.get_rule_list ("RejectHeader") + if rej_headers: + rej_headers = rej_headers.headers + for header_line in rej_headers: + header_re = self.headers.get (header_line) + if header_re is not None and header_re == rej_headers[header_line]: + self.send_error (400, 'Blacklisted Header was Sent') + self.end_headers () + return False + return True + + def __log_request (self, method): + req = method + " " + self.path + self.server.request_headers.append (req) + + def send_head (self, method): + """ Common code for GET and HEAD Commands. + This method is overriden to use the fileSys dict. + + The method variable contains whether this was a HEAD or a GET Request. + According to RFC 2616, the server should not differentiate between + the two requests, however, we use it here for a specific test. + """ + + if self.path == "/": + path = "index.html" + else: + path = self.path[1:] + + self.__log_request (method) + + if path in self.server.fileSys: + self.rules = self.server.server_configs.get (path) + + for rule_name in self.rules: + try: + assert hasattr (self, rule_name) + getattr (self, rule_name) (self.rules [rule_name]) + except AssertionError as ae: + msg = "Method " + rule_name + " not defined" + self.send_error (500, msg) + return (None, None) + except ServerError as se: + print (se.__str__()) + return (None, None) + + content = self.server.fileSys.get (path) + content_length = len (content) + try: + self.range_begin = self.parse_range_header ( + self.headers.get ("Range"), content_length) + except InvalidRangeHeader as ae: + # self.log_error("%s", ae.err_message) + if ae.err_message == "Range Overflow": + self.send_response (416) + self.finish_headers () + return (None, None) + else: + self.range_begin = None + if self.range_begin is None: + self.send_response (200) + else: + self.send_response (206) + self.send_header ("Accept-Ranges", "bytes") + self.send_header ("Content-Range", + "bytes %d-%d/%d" % (self.range_begin, + content_length - 1, + content_length)) + content_length -= self.range_begin + cont_type = self.guess_type (path) + self.send_header ("Content-type", cont_type) + self.send_header ("Content-Length", content_length) + self.finish_headers () + return (content, self.range_begin) + else: + self.send_error (404, "Not Found") + return (None, None) + + def guess_type (self, path): + base_name = basename ("/" + path) + name, ext = splitext (base_name) + extension_map = { + ".txt" : "text/plain", + ".css" : "text/css", + ".html" : "text/html" + } + if ext in extension_map: + return extension_map[ext] + else: + return "text/plain" + + +class HTTPd (threading.Thread): + server_class = StoppableHTTPServer + handler = _Handler + def __init__ (self, addr=None): + threading.Thread.__init__ (self) + if addr is None: + addr = ('localhost', 0) + self.server_inst = self.server_class (addr, self.handler) + self.server_address = self.server_inst.socket.getsockname()[:2] + + def run (self): + self.server_inst.serve_forever () + + def config(self, file_list, server_rules): + self.server_inst.config(file_list, server_rules) + + def server_sett (self, settings): + self.server_inst.server_sett (settings) + +class HTTPSd (HTTPd): + + server_class = HTTPSServer + +# vim: set ts=4 sts=4 sw=4 tw=80 et : diff --git a/testenv/test/__init__.py b/testenv/test/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/testenv/test/__init__.py @@ -0,0 +1 @@ + diff --git a/testenv/test/base_test.py b/testenv/test/base_test.py new file mode 100644 index 0000000..c0996d7 --- /dev/null +++ b/testenv/test/base_test.py @@ -0,0 +1,247 @@ +import os +import shlex +import shutil +from subprocess import call +import time +import re +import traceback +import conf +from exc.test_failed import TestFailed +from misc.colour_terminal import print_blue, print_red + + +class BaseTest: + + """ + Class that defines methods common to both HTTP and FTP Tests. + Note that this is an abstract class, subclasses must implement + * stop_server() + * instantiate_server_by(protocol) + """ + + def __init__(self, name, pre_hook, test_params, post_hook, protocols): + """ + Define the class-wide variables (or attributes). + Attributes should not be defined outside __init__. + """ + self.name = name + self.pre_configs = pre_hook or {} # if pre_hook == None, then + # {} (an empty dict object) is + # passed to self.pre_configs + self.test_params = test_params or {} + self.post_configs = post_hook or {} + self.protocols = protocols + + self.servers = [] + self.domains = [] + self.port = -1 + + self.wget_options = '' + self.urls = [] + + self.tests_passed = True + self.init_test_env() + + self.ret_code = 0 + + + def get_test_dir(self): + return self.name + '-test' + + + def init_test_env (self): + test_dir = self.get_test_dir() + try: + os.mkdir(test_dir) + except FileExistsError: + shutil.rmtree(test_dir) + os.mkdir(test_dir) + os.chdir(test_dir) + + + def do_test(self): + self.pre_hook_call() + self.call_test() + self.post_hook_call() + + + def hook_call(self, configs, name): + for conf_name, conf_arg in configs.items(): + try: + # conf.find_conf(conf_name) returns the required conf class, + # then the class is instantiated with conf_arg, then the + # conf instance is called with this test instance itself to + # invoke the desired hook + conf.find_conf(conf_name)(conf_arg)(self) + except AttributeError: + self.stop_server() + raise TestFailed("%s %s not defined." % + (name, conf_name)) + + + def pre_hook_call(self): + self.hook_call(self.pre_configs, 'Pre Test Function') + + + def post_hook_call(self): + self.hook_call(self.post_configs, 'Post Test Function') + + + def stop_server(self): + """ + Subclasses must implement this method in order to stop certain + servers of different types. + """ + raise NotImplementedError + + + def call_test(self): + self.hook_call(self.test_params, 'Test Option') + + try: + self.ret_code = self.exec_wget() + except TestFailed as e: + raise e + finally: + self.stop_server() + + + def server_setup(self): + print_blue("Running Test %s" % self.name) + for protocol in self.protocols: + instance = self.instantiate_server_by(protocol) + self.servers.append(instance) + + # servers instantiated by different protocols may differ in + # ports and etc. + # so we should record different domains respect to servers. + domain = self.get_domain_addr(instance.server_address) + self.domains.append(domain) + + + def instantiate_server_by(self, protocol): + """ + Subclasses must override this method to actually instantiate servers + for test cases. + """ + raise NotImplementedError + + + def get_domain_addr(self, addr): + # TODO if there's a multiple number of ports, wouldn't it be + # overridden to the port of the last invocation? + self.port = str(addr[1]) + + return '%s:%s' % (addr[0], self.port) + + + def exec_wget(self): + cmd_line = self.gen_cmd_line() + params = shlex.split(cmd_line) + print(params) + + if os.getenv("SERVER_WAIT"): + time.sleep(float(os.getenv("SERVER_WAIT"))) + + try: + ret_code = call(params) + except FileNotFoundError: + raise TestFailed("The Wget Executable does not exist at the " + "expected path.") + + return ret_code + + + def gen_cmd_line(self): + test_path = os.path.abspath(".") + wget_path = os.path.abspath(os.path.join(test_path, + "..", '..', 'src', "wget")) + + cmd_line = '%s %s ' % (wget_path, self.wget_options) + for protocol, urls, domain in zip(self.protocols, + self.urls, + self.domains): + # zip is function for iterating multiple lists at the same time. + # e.g. for item1, item2 in zip([1, 2, 3, 4, 5], + # ['a', 'b', 'c', 'd', 'e']): + # print(item1, item2) + # generates the following output: + # 1 a + # 2 b + # 3 c + # 4 d + # 5 e + protocol = protocol.lower() + for url in urls: + cmd_line += '%s://%s/%s ' % (protocol, domain, url) + + print(cmd_line) + + return cmd_line + + + def __test_cleanup (self): + os.chdir('..') + try: + if not os.getenv("NO_CLEANUP"): + shutil.rmtree(self.get_test_dir()) + except: + print("Unknown Exception while trying to remove Test Environment.") + + + def begin (self): + return 0 if self.tests_passed else 100 + + + def replace_quotes(self, string): + pattern = re.compile ('\{\{\w+\}\}') + match_obj = pattern.search(string) + if match_obj is not None: + rep = str(match_obj.group()) + temp = getattr(self, rep.strip('{}')) + string = string.replace(rep, temp) + return string + + + @staticmethod + def get_server_rules(file_obj): + """ + The handling of expect header could be made much better when the + options are parsed in a true and better fashion. For an example, + see the commented portion in Test-basic-auth.py. + """ + server_rules = {} + for rule_name, rule in file_obj.rules.items(): + server_rules[rule_name] = conf.find_conf(rule_name)(rule) + return server_rules + + + def __enter__(self): + """ + Initialization for with statement. + """ + return self + + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + If the with statement got executed with no exception raised, then + exc_type, exc_val, exc_tb are all None. + """ + if exc_val: + self.tests_passed = False + if exc_type is TestFailed: + print_red('Error: %s.' % exc_val.error) + else: + print_red('Unhandled exception caught.') + print(exc_val) + traceback.print_tb(exc_tb) + self.__test_cleanup() + + return True + + + # `ServerConf' is actually never referenced. + # def ServerConf (self, server_settings): + # for i in range (0, self.servers): + # self.server_list[i].server_sett (server_settings) diff --git a/testenv/test/http_test.py b/testenv/test/http_test.py new file mode 100644 index 0000000..1ccbe35 --- /dev/null +++ b/testenv/test/http_test.py @@ -0,0 +1,53 @@ +from misc.colour_terminal import print_green +from misc.constants import HTTP, HTTPS +from server.http.http_server import HTTPd, HTTPSd +from test.base_test import BaseTest + + +class HTTPTest(BaseTest): + + """ Class for HTTP Tests. """ + + # Temp Notes: It is expected that when pre-hook functions are executed, only an empty test-dir exists. + # pre-hook functions are executed just prior to the call to Wget is made. + # post-hook functions will be executed immediately after the call to Wget returns. + + def __init__(self, + name="Unnamed Test", + pre_hook=None, + test_params=None, + post_hook=None, + protocols=(HTTP,)): + super(HTTPTest, self).__init__(name, + pre_hook, + test_params, + post_hook, + protocols) + with self: + # if any exception occurs, self.__exit__ will be immediately + # called + self.server_setup() + self.do_test() + print_green('Test Passed.') + + + def instantiate_server_by(self, protocol): + server = {HTTP: HTTPd, + HTTPS: HTTPSd}[protocol]() + server.start() + + return server + + + def request_remaining(self): + return [s.server_inst.get_req_headers() + for s in self.servers] + + + def stop_server(self): + for server in self.servers: + server.server_inst.shutdown() + + + +# vim: set ts=4 sts=4 sw=4 tw=80 et : -- 1.8.3.2