From dfbc8e2ec8ab159ff7231cf070378dddc96a75b0 Mon Sep 17 00:00:00 2001
From: Daniel Blixt <daniel.blixt.2@volvocars.com>
Date: Tue, 19 Jan 2021 16:14:11 +0100
Subject: [PATCH] Use urlencoded filenames in test fixtures
When checking out repo to Windows environment, some files in test
fixture file trees do not get checked out. By setting renaming those
files with an urlencoded name, and populating a temporary directory
with corresponding file structure and decoded filenames during test by
a fixtures script, this will not cause checkout problems for those who
want to read the repo in Win environment
Change-Id: If354eaf3f0bf2e86ddc05e3750aa5f0067dd9e21
---
.../library/test-fixtures/.gitattributes | 1 +
.../library/filefixture.py | 139 ++++++
.../library/test-fixtures/.gitattributes | 1 +
...=> %E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt} | 0
.../subdir/{foo::3.txt => foo%3A%3A3.txt} | 0
.../library/test_zuul_swift_upload.py | 422 +++++++++--------
roles/upload-logs-base/library/filefixture.py | 139 ++++++
.../library/test-fixtures/.gitattributes | 1 +
...=> %E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt} | 0
.../subdir/{foo::3.txt => foo%3A%3A3.txt} | 0
roles/upload-logs-base/library/test_index.py | 437 +++++++++---------
11 files changed, 734 insertions(+), 406 deletions(-)
create mode 100644 roles/generate-zuul-manifest/library/test-fixtures/.gitattributes
create mode 100644 roles/test-upload-logs-swift/library/filefixture.py
create mode 100644 roles/test-upload-logs-swift/library/test-fixtures/.gitattributes
rename roles/test-upload-logs-swift/library/test-fixtures/logs/{Ꮓບບξ-unicode.txt => %E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt} (100%)
rename roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/{foo::3.txt => foo%3A%3A3.txt} (100%)
create mode 100644 roles/upload-logs-base/library/filefixture.py
create mode 100644 roles/upload-logs-base/library/test-fixtures/.gitattributes
rename roles/upload-logs-base/library/test-fixtures/logs/{Ꮓບບξ-unicode.txt => %E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt} (100%)
rename roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/{foo::3.txt => foo%3A%3A3.txt} (100%)
diff --git a/roles/generate-zuul-manifest/library/test-fixtures/.gitattributes b/roles/generate-zuul-manifest/library/test-fixtures/.gitattributes
new file mode 100644
index 000000000..2325d0821
--- /dev/null
+++ b/roles/generate-zuul-manifest/library/test-fixtures/.gitattributes
@@ -0,0 +1 @@
+*.* eol=lf
\ No newline at end of file
diff --git a/roles/test-upload-logs-swift/library/filefixture.py b/roles/test-upload-logs-swift/library/filefixture.py
new file mode 100644
index 000000000..615b26ab1
--- /dev/null
+++ b/roles/test-upload-logs-swift/library/filefixture.py
@@ -0,0 +1,139 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Handle file name special characters in a file tree.
+
+All files stored in a filetree can be renamed to urlencoded filenames.
+A file tree can also be copied to a temporary location with file names
+decoded, to be used in tests with special characters that are not always
+possible to store on all file systems.
+
+"""
+
+from __future__ import print_function
+
+import os
+try:
+ from urllib.parse import quote as urlib_quote
+ from urllib.parse import unquote as urlib_unquote
+except ImportError:
+ from urllib import quote as urlib_quote
+ from urllib import unquote as urlib_unquote
+import argparse
+import fixtures
+import tempfile
+import shutil
+
+
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+ 'test-fixtures')
+
+SAFE_CHARS = "\\/"
+
+
+def portable_makedirs_exist_ok(path):
+ try:
+ os.makedirs(path, exist_ok=True)
+ except TypeError as err:
+ if "unexpected keyword argument" not in str(err):
+ raise err
+ if not os.path.exists(path):
+ try:
+ os.makedirs(path)
+ except OSError as err:
+ if "File exists" not in err:
+ raise err
+
+
+def urlencode_filetree():
+ for root, _, files in os.walk(FIXTURE_DIR):
+ for filename in files:
+ os.rename(
+ os.path.join(root, filename),
+ os.path.join(
+ root, urlib_quote(urlib_unquote(filename), SAFE_CHARS)
+ )
+ )
+
+
+def populate_filetree(dst_dir=None):
+
+ if not os.path.exists(FIXTURE_DIR):
+ return None
+
+ if not dst_dir:
+ dst_dir = tempfile.mkdtemp()
+
+ portable_makedirs_exist_ok(dst_dir)
+
+ for root, dirs, files in os.walk(FIXTURE_DIR):
+ dst_root = root.replace(FIXTURE_DIR, dst_dir, 1)
+ for directory in dirs:
+ portable_makedirs_exist_ok(os.path.join(dst_root, directory))
+ for filename in files:
+ try:
+ shutil.copyfile(
+ os.path.join(root, filename),
+ os.path.join(dst_root, urlib_unquote(filename))
+ )
+ except IOError as err:
+ print(
+ "\nFile {}".format(
+ os.path.join(dst_root, urlib_unquote(filename))
+ ),
+ "\nnot possible to write to disk,",
+ "\npossibly due to filename not being valid on Windows?\n"
+ )
+ shutil.rmtree(dst_dir)
+ raise err
+
+ return dst_dir
+
+
+class FileFixture(fixtures.Fixture):
+
+ def _setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(self.local_clean_up)
+ populate_filetree(self.root)
+ # There is no cleanup action, as the filetree is left intact for other
+ # tests to use
+
+ def local_clean_up(self):
+ shutil.rmtree(self.root)
+
+
+if __name__ == '__main__':
+
+ parser = argparse.ArgumentParser(__doc__)
+ parser.add_argument(
+ '--populate',
+ help="Causes files in {}".format(FIXTURE_DIR) +
+ "to be copied with decoded file name to a tmp dir" +
+ "Overrides --encode",
+ action='store_true'
+ )
+ parser.add_argument(
+ '--encode',
+ help="Causes files under {} to be renamed with urlencoding.".format(
+ FIXTURE_DIR
+ ) + "DEFAULT behaviour, overridden by --populate",
+ action='store_true'
+ )
+ args = parser.parse_args()
+
+ if args.populate:
+ print(populate_filetree())
+ else:
+ urlencode_filetree()
diff --git a/roles/test-upload-logs-swift/library/test-fixtures/.gitattributes b/roles/test-upload-logs-swift/library/test-fixtures/.gitattributes
new file mode 100644
index 000000000..2325d0821
--- /dev/null
+++ b/roles/test-upload-logs-swift/library/test-fixtures/.gitattributes
@@ -0,0 +1 @@
+*.* eol=lf
\ No newline at end of file
diff --git a/roles/test-upload-logs-swift/library/test-fixtures/logs/Ꮓບບξ-unicode.txt b/roles/test-upload-logs-swift/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt
similarity index 100%
rename from roles/test-upload-logs-swift/library/test-fixtures/logs/Ꮓບບξ-unicode.txt
rename to roles/test-upload-logs-swift/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt
diff --git a/roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo::3.txt b/roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt
similarity index 100%
rename from roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo::3.txt
rename to roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt
diff --git a/roles/test-upload-logs-swift/library/test_zuul_swift_upload.py b/roles/test-upload-logs-swift/library/test_zuul_swift_upload.py
index cf088987a..2e0c32a23 100644
--- a/roles/test-upload-logs-swift/library/test_zuul_swift_upload.py
+++ b/roles/test-upload-logs-swift/library/test_zuul_swift_upload.py
@@ -32,7 +32,7 @@ except ImportError:
import requests
from bs4 import BeautifulSoup
from .zuul_swift_upload import FileList, Indexer, FileDetail, Uploader
-
+from .filefixture import FileFixture
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'test-fixtures')
@@ -50,8 +50,11 @@ class SymlinkFixture(fixtures.Fixture):
]
def _setUp(self):
+ self.file_fixture = FileFixture()
+ self.file_fixture.setUp()
+ self.addCleanup(self.file_fixture.cleanUp)
for (src, target) in self.links:
- path = os.path.join(FIXTURE_DIR, 'links', src)
+ path = os.path.join(self.file_fixture.root, 'links', src)
os.symlink(target, path)
self.addCleanup(os.unlink, path)
@@ -88,66 +91,69 @@ class TestFileList(testtools.TestCase):
'''Test a single directory with a trailing slash'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('controller', 'application/directory', None),
- ('zuul-info', 'application/directory', None),
- ('job-output.json', 'application/json', None),
- (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
- 'text/plain', None),
- ('controller/subdir', 'application/directory', None),
- ('controller/compressed.gz', 'text/plain', 'gzip'),
- ('controller/cpu-load.svg', 'image/svg+xml', None),
- ('controller/journal.xz', 'text/plain', 'xz'),
- ('controller/service_log.txt', 'text/plain', None),
- ('controller/syslog', 'text/plain', None),
- ('controller/subdir/foo::3.txt', 'text/plain', None),
- ('controller/subdir/subdir.txt', 'text/plain', None),
- ('zuul-info/inventory.yaml', 'text/plain', None),
- ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
- ])
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root, 'logs/'))
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('controller', 'application/directory', None),
+ ('zuul-info', 'application/directory', None),
+ ('job-output.json', 'application/json', None),
+ (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
+ 'text/plain', None),
+ ('controller/subdir', 'application/directory', None),
+ ('controller/compressed.gz', 'text/plain', 'gzip'),
+ ('controller/cpu-load.svg', 'image/svg+xml', None),
+ ('controller/journal.xz', 'text/plain', 'xz'),
+ ('controller/service_log.txt', 'text/plain', None),
+ ('controller/syslog', 'text/plain', None),
+ ('controller/subdir/foo::3.txt', 'text/plain', None),
+ ('controller/subdir/subdir.txt', 'text/plain', None),
+ ('zuul-info/inventory.yaml', 'text/plain', None),
+ ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+ ])
def test_single_dir(self):
'''Test a single directory without a trailing slash'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs'))
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('logs', 'application/directory', None),
- ('logs/controller', 'application/directory', None),
- ('logs/zuul-info', 'application/directory', None),
- ('logs/job-output.json', 'application/json', None),
- (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
- 'text/plain', None),
- ('logs/controller/subdir', 'application/directory', None),
- ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
- ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
- ('logs/controller/journal.xz', 'text/plain', 'xz'),
- ('logs/controller/service_log.txt', 'text/plain', None),
- ('logs/controller/syslog', 'text/plain', None),
- ('logs/controller/subdir/foo::3.txt', 'text/plain', None),
- ('logs/controller/subdir/subdir.txt', 'text/plain', None),
- ('logs/zuul-info/inventory.yaml', 'text/plain', None),
- ('logs/zuul-info/zuul-info.controller.txt',
- 'text/plain', None),
- ])
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root, 'logs'))
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('logs', 'application/directory', None),
+ ('logs/controller', 'application/directory', None),
+ ('logs/zuul-info', 'application/directory', None),
+ ('logs/job-output.json', 'application/json', None),
+ (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
+ 'text/plain', None),
+ ('logs/controller/subdir', 'application/directory', None),
+ ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
+ ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
+ ('logs/controller/journal.xz', 'text/plain', 'xz'),
+ ('logs/controller/service_log.txt', 'text/plain', None),
+ ('logs/controller/syslog', 'text/plain', None),
+ ('logs/controller/subdir/foo::3.txt', 'text/plain', None),
+ ('logs/controller/subdir/subdir.txt', 'text/plain', None),
+ ('logs/zuul-info/inventory.yaml', 'text/plain', None),
+ ('logs/zuul-info/zuul-info.controller.txt',
+ 'text/plain', None),
+ ])
def test_single_file(self):
'''Test a single file'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR,
- 'logs/zuul-info/inventory.yaml'))
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('inventory.yaml', 'text/plain', None),
- ])
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root,
+ 'logs/zuul-info/inventory.yaml'))
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('inventory.yaml', 'text/plain', None),
+ ])
def test_symlinks(self):
'''Test symlinks'''
with FileList() as fl:
- self.useFixture(SymlinkFixture())
- fl.add(os.path.join(FIXTURE_DIR, 'links/'))
+ symlink_fixture = self.useFixture(SymlinkFixture())
+ fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/'))
self.assert_files(fl, [
('', 'application/directory', None),
('controller', 'application/directory', None),
@@ -165,7 +171,8 @@ class TestFileList(testtools.TestCase):
def test_index_files(self):
'''Test index generation'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs'))
+ symlink_fixture = self.useFixture(SymlinkFixture())
+ fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs'))
ix = Indexer(fl)
ix.make_indexes()
@@ -223,32 +230,33 @@ class TestFileList(testtools.TestCase):
def test_index_files_trailing_slash(self):
'''Test index generation with a trailing slash'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
- ix = Indexer(fl)
- ix.make_indexes()
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root, 'logs/'))
+ ix = Indexer(fl)
+ ix.make_indexes()
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('controller', 'application/directory', None),
- ('zuul-info', 'application/directory', None),
- ('job-output.json', 'application/json', None),
- (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
- 'text/plain', None),
- ('index.html', 'text/html', None),
- ('controller/subdir', 'application/directory', None),
- ('controller/compressed.gz', 'text/plain', 'gzip'),
- ('controller/cpu-load.svg', 'image/svg+xml', None),
- ('controller/journal.xz', 'text/plain', 'xz'),
- ('controller/service_log.txt', 'text/plain', None),
- ('controller/syslog', 'text/plain', None),
- ('controller/index.html', 'text/html', None),
- ('controller/subdir/foo::3.txt', 'text/plain', None),
- ('controller/subdir/subdir.txt', 'text/plain', None),
- ('controller/subdir/index.html', 'text/html', None),
- ('zuul-info/inventory.yaml', 'text/plain', None),
- ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
- ('zuul-info/index.html', 'text/html', None),
- ])
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('controller', 'application/directory', None),
+ ('zuul-info', 'application/directory', None),
+ ('job-output.json', 'application/json', None),
+ (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
+ 'text/plain', None),
+ ('index.html', 'text/html', None),
+ ('controller/subdir', 'application/directory', None),
+ ('controller/compressed.gz', 'text/plain', 'gzip'),
+ ('controller/cpu-load.svg', 'image/svg+xml', None),
+ ('controller/journal.xz', 'text/plain', 'xz'),
+ ('controller/service_log.txt', 'text/plain', None),
+ ('controller/syslog', 'text/plain', None),
+ ('controller/index.html', 'text/html', None),
+ ('controller/subdir/foo::3.txt', 'text/plain', None),
+ ('controller/subdir/subdir.txt', 'text/plain', None),
+ ('controller/subdir/index.html', 'text/html', None),
+ ('zuul-info/inventory.yaml', 'text/plain', None),
+ ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+ ('zuul-info/index.html', 'text/html', None),
+ ])
top_index = self.find_file(fl, 'index.html')
page = open(top_index.full_path).read()
@@ -280,134 +288,145 @@ class TestFileList(testtools.TestCase):
def test_topdir_parent_link(self):
'''Test index generation creates topdir parent link'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
- ix = Indexer(fl)
- ix.make_indexes(
- create_parent_links=True,
- create_topdir_parent_link=True)
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root, 'logs/'))
+ ix = Indexer(fl)
+ ix.make_indexes(
+ create_parent_links=True,
+ create_topdir_parent_link=True)
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('controller', 'application/directory', None),
- ('zuul-info', 'application/directory', None),
- ('job-output.json', 'application/json', None),
- (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
- 'text/plain', None),
- ('index.html', 'text/html', None),
- ('controller/subdir', 'application/directory', None),
- ('controller/compressed.gz', 'text/plain', 'gzip'),
- ('controller/cpu-load.svg', 'image/svg+xml', None),
- ('controller/journal.xz', 'text/plain', 'xz'),
- ('controller/service_log.txt', 'text/plain', None),
- ('controller/syslog', 'text/plain', None),
- ('controller/index.html', 'text/html', None),
- ('controller/subdir/foo::3.txt', 'text/plain', None),
- ('controller/subdir/subdir.txt', 'text/plain', None),
- ('controller/subdir/index.html', 'text/html', None),
- ('zuul-info/inventory.yaml', 'text/plain', None),
- ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
- ('zuul-info/index.html', 'text/html', None),
- ])
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('controller', 'application/directory', None),
+ ('zuul-info', 'application/directory', None),
+ ('job-output.json', 'application/json', None),
+ (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
+ 'text/plain', None),
+ ('index.html', 'text/html', None),
+ ('controller/subdir', 'application/directory', None),
+ ('controller/compressed.gz', 'text/plain', 'gzip'),
+ ('controller/cpu-load.svg', 'image/svg+xml', None),
+ ('controller/journal.xz', 'text/plain', 'xz'),
+ ('controller/service_log.txt', 'text/plain', None),
+ ('controller/syslog', 'text/plain', None),
+ ('controller/index.html', 'text/html', None),
+ ('controller/subdir/foo::3.txt', 'text/plain', None),
+ ('controller/subdir/subdir.txt', 'text/plain', None),
+ ('controller/subdir/index.html', 'text/html', None),
+ ('zuul-info/inventory.yaml', 'text/plain', None),
+ ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+ ('zuul-info/index.html', 'text/html', None),
+ ])
- top_index = self.find_file(fl, 'index.html')
- page = open(top_index.full_path).read()
- page = BeautifulSoup(page, 'html.parser')
- rows = page.find_all('tr')[1:]
+ top_index = self.find_file(fl, 'index.html')
+ page = open(top_index.full_path).read()
+ page = BeautifulSoup(page, 'html.parser')
+ rows = page.find_all('tr')[1:]
- self.assertEqual(len(rows), 5)
+ self.assertEqual(len(rows), 5)
- self.assertEqual(rows[0].find('a').get('href'), '../')
- self.assertEqual(rows[0].find('a').text, '../')
+ self.assertEqual(rows[0].find('a').get('href'), '../')
+ self.assertEqual(rows[0].find('a').text, '../')
- self.assertEqual(rows[1].find('a').get('href'), 'controller/')
- self.assertEqual(rows[1].find('a').text, 'controller/')
+ self.assertEqual(rows[1].find('a').get('href'), 'controller/')
+ self.assertEqual(rows[1].find('a').text, 'controller/')
- self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
- self.assertEqual(rows[2].find('a').text, 'zuul-info/')
+ self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
+ self.assertEqual(rows[2].find('a').text, 'zuul-info/')
- subdir_index = self.find_file(fl, 'controller/subdir/index.html')
- page = open(subdir_index.full_path).read()
- page = BeautifulSoup(page, 'html.parser')
- rows = page.find_all('tr')[1:]
- self.assertEqual(rows[0].find('a').get('href'), '../')
- self.assertEqual(rows[0].find('a').text, '../')
+ subdir_index = self.find_file(
+ fl, 'controller/subdir/index.html'
+ )
+ page = open(subdir_index.full_path).read()
+ page = BeautifulSoup(page, 'html.parser')
+ rows = page.find_all('tr')[1:]
+ self.assertEqual(rows[0].find('a').get('href'), '../')
+ self.assertEqual(rows[0].find('a').text, '../')
- # Test proper escaping of files with funny names
- self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
- self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
- # Test files without escaping
- self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
- self.assertEqual(rows[2].find('a').text, 'subdir.txt')
+ # Test proper escaping of files with funny names
+ self.assertEqual(
+ rows[1].find('a').get('href'), 'foo%3A%3A3.txt'
+ )
+ self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
+ # Test files without escaping
+ self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
+ self.assertEqual(rows[2].find('a').text, 'subdir.txt')
def test_no_parent_links(self):
'''Test index generation creates topdir parent link'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
- ix = Indexer(fl)
- ix.make_indexes(
- create_parent_links=False,
- create_topdir_parent_link=False)
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root, 'logs/'))
+ ix = Indexer(fl)
+ ix.make_indexes(
+ create_parent_links=False,
+ create_topdir_parent_link=False)
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('controller', 'application/directory', None),
- ('zuul-info', 'application/directory', None),
- ('job-output.json', 'application/json', None),
- (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
- 'text/plain', None),
- ('index.html', 'text/html', None),
- ('controller/subdir', 'application/directory', None),
- ('controller/compressed.gz', 'text/plain', 'gzip'),
- ('controller/cpu-load.svg', 'image/svg+xml', None),
- ('controller/journal.xz', 'text/plain', 'xz'),
- ('controller/service_log.txt', 'text/plain', None),
- ('controller/syslog', 'text/plain', None),
- ('controller/index.html', 'text/html', None),
- ('controller/subdir/foo::3.txt', 'text/plain', None),
- ('controller/subdir/subdir.txt', 'text/plain', None),
- ('controller/subdir/index.html', 'text/html', None),
- ('zuul-info/inventory.yaml', 'text/plain', None),
- ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
- ('zuul-info/index.html', 'text/html', None),
- ])
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('controller', 'application/directory', None),
+ ('zuul-info', 'application/directory', None),
+ ('job-output.json', 'application/json', None),
+ (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
+ 'text/plain', None),
+ ('index.html', 'text/html', None),
+ ('controller/subdir', 'application/directory', None),
+ ('controller/compressed.gz', 'text/plain', 'gzip'),
+ ('controller/cpu-load.svg', 'image/svg+xml', None),
+ ('controller/journal.xz', 'text/plain', 'xz'),
+ ('controller/service_log.txt', 'text/plain', None),
+ ('controller/syslog', 'text/plain', None),
+ ('controller/index.html', 'text/html', None),
+ ('controller/subdir/foo::3.txt', 'text/plain', None),
+ ('controller/subdir/subdir.txt', 'text/plain', None),
+ ('controller/subdir/index.html', 'text/html', None),
+ ('zuul-info/inventory.yaml', 'text/plain', None),
+ ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+ ('zuul-info/index.html', 'text/html', None),
+ ])
- top_index = self.find_file(fl, 'index.html')
- page = open(top_index.full_path).read()
- page = BeautifulSoup(page, 'html.parser')
- rows = page.find_all('tr')[1:]
+ top_index = self.find_file(fl, 'index.html')
+ page = open(top_index.full_path).read()
+ page = BeautifulSoup(page, 'html.parser')
+ rows = page.find_all('tr')[1:]
- self.assertEqual(len(rows), 4)
+ self.assertEqual(len(rows), 4)
- self.assertEqual(rows[0].find('a').get('href'), 'controller/')
- self.assertEqual(rows[0].find('a').text, 'controller/')
+ self.assertEqual(rows[0].find('a').get('href'), 'controller/')
+ self.assertEqual(rows[0].find('a').text, 'controller/')
- self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
- self.assertEqual(rows[1].find('a').text, 'zuul-info/')
+ self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
+ self.assertEqual(rows[1].find('a').text, 'zuul-info/')
- subdir_index = self.find_file(fl, 'controller/subdir/index.html')
- page = open(subdir_index.full_path).read()
- page = BeautifulSoup(page, 'html.parser')
- rows = page.find_all('tr')[1:]
+ subdir_index = self.find_file(
+ fl, 'controller/subdir/index.html'
+ )
+ page = open(subdir_index.full_path).read()
+ page = BeautifulSoup(page, 'html.parser')
+ rows = page.find_all('tr')[1:]
- # Test proper escaping of files with funny names
- self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt')
- self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
- # Test files without escaping
- self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
- self.assertEqual(rows[1].find('a').text, 'subdir.txt')
+ # Test proper escaping of files with funny names
+ self.assertEqual(
+ rows[0].find('a').get('href'), 'foo%3A%3A3.txt'
+ )
+ self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
+ # Test files without escaping
+ self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
+ self.assertEqual(rows[1].find('a').text, 'subdir.txt')
class TestFileDetail(testtools.TestCase):
def test_get_file_detail(self):
'''Test files info'''
- path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
- file_detail = FileDetail(path, '')
- path_stat = os.stat(path)
- self.assertEqual(
- time.gmtime(path_stat[stat.ST_MTIME]),
- file_detail.last_modified)
- self.assertEqual(16, file_detail.size)
+ with FileFixture() as file_fixture:
+ path = os.path.join(file_fixture.root, 'logs/job-output.json')
+ file_detail = FileDetail(path, '')
+ path_stat = os.stat(path)
+ self.assertEqual(
+ time.gmtime(path_stat[stat.ST_MTIME]),
+ file_detail.last_modified)
+ self.assertEqual(16, file_detail.size)
def test_get_file_detail_missing_file(self):
'''Test files that go missing during a walk'''
@@ -447,26 +466,29 @@ class TestUpload(testtools.TestCase):
)
# Get some test files to upload
- files = [
- FileDetail(
- os.path.join(FIXTURE_DIR, "logs/job-output.json"),
- "job-output.json",
- ),
- FileDetail(
- os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"),
- "inventory.yaml",
- ),
- ]
-
- expected_failures = [
- {
- "file": "job-output.json",
- "error": (
- "Error posting file after multiple attempts: "
- "Failed for a reason"
+ with FileFixture() as file_fixture:
+ files = [
+ FileDetail(
+ os.path.join(file_fixture.root, "logs/job-output.json"),
+ "job-output.json",
),
- },
- ]
+ FileDetail(
+ os.path.join(
+ file_fixture.root, "logs/zuul-info/inventory.yaml"
+ ),
+ "inventory.yaml",
+ ),
+ ]
- failures = uploader.upload(files)
- self.assertEqual(expected_failures, failures)
+ expected_failures = [
+ {
+ "file": "job-output.json",
+ "error": (
+ "Error posting file after multiple attempts: "
+ "Failed for a reason"
+ ),
+ },
+ ]
+
+ failures = uploader.upload(files)
+ self.assertEqual(expected_failures, failures)
diff --git a/roles/upload-logs-base/library/filefixture.py b/roles/upload-logs-base/library/filefixture.py
new file mode 100644
index 000000000..615b26ab1
--- /dev/null
+++ b/roles/upload-logs-base/library/filefixture.py
@@ -0,0 +1,139 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Handle file name special characters in a file tree.
+
+All files stored in a filetree can be renamed to urlencoded filenames.
+A file tree can also be copied to a temporary location with file names
+decoded, to be used in tests with special characters that are not always
+possible to store on all file systems.
+
+"""
+
+from __future__ import print_function
+
+import os
+try:
+ from urllib.parse import quote as urlib_quote
+ from urllib.parse import unquote as urlib_unquote
+except ImportError:
+ from urllib import quote as urlib_quote
+ from urllib import unquote as urlib_unquote
+import argparse
+import fixtures
+import tempfile
+import shutil
+
+
+FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
+ 'test-fixtures')
+
+SAFE_CHARS = "\\/"
+
+
+def portable_makedirs_exist_ok(path):
+ try:
+ os.makedirs(path, exist_ok=True)
+ except TypeError as err:
+ if "unexpected keyword argument" not in str(err):
+ raise err
+ if not os.path.exists(path):
+ try:
+ os.makedirs(path)
+ except OSError as err:
+ if "File exists" not in err:
+ raise err
+
+
+def urlencode_filetree():
+ for root, _, files in os.walk(FIXTURE_DIR):
+ for filename in files:
+ os.rename(
+ os.path.join(root, filename),
+ os.path.join(
+ root, urlib_quote(urlib_unquote(filename), SAFE_CHARS)
+ )
+ )
+
+
+def populate_filetree(dst_dir=None):
+
+ if not os.path.exists(FIXTURE_DIR):
+ return None
+
+ if not dst_dir:
+ dst_dir = tempfile.mkdtemp()
+
+ portable_makedirs_exist_ok(dst_dir)
+
+ for root, dirs, files in os.walk(FIXTURE_DIR):
+ dst_root = root.replace(FIXTURE_DIR, dst_dir, 1)
+ for directory in dirs:
+ portable_makedirs_exist_ok(os.path.join(dst_root, directory))
+ for filename in files:
+ try:
+ shutil.copyfile(
+ os.path.join(root, filename),
+ os.path.join(dst_root, urlib_unquote(filename))
+ )
+ except IOError as err:
+ print(
+ "\nFile {}".format(
+ os.path.join(dst_root, urlib_unquote(filename))
+ ),
+ "\nnot possible to write to disk,",
+ "\npossibly due to filename not being valid on Windows?\n"
+ )
+ shutil.rmtree(dst_dir)
+ raise err
+
+ return dst_dir
+
+
+class FileFixture(fixtures.Fixture):
+
+ def _setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(self.local_clean_up)
+ populate_filetree(self.root)
+ # There is no cleanup action, as the filetree is left intact for other
+ # tests to use
+
+ def local_clean_up(self):
+ shutil.rmtree(self.root)
+
+
+if __name__ == '__main__':
+
+ parser = argparse.ArgumentParser(__doc__)
+ parser.add_argument(
+ '--populate',
+ help="Causes files in {}".format(FIXTURE_DIR) +
+ "to be copied with decoded file name to a tmp dir" +
+ "Overrides --encode",
+ action='store_true'
+ )
+ parser.add_argument(
+ '--encode',
+ help="Causes files under {} to be renamed with urlencoding.".format(
+ FIXTURE_DIR
+ ) + "DEFAULT behaviour, overridden by --populate",
+ action='store_true'
+ )
+ args = parser.parse_args()
+
+ if args.populate:
+ print(populate_filetree())
+ else:
+ urlencode_filetree()
diff --git a/roles/upload-logs-base/library/test-fixtures/.gitattributes b/roles/upload-logs-base/library/test-fixtures/.gitattributes
new file mode 100644
index 000000000..2325d0821
--- /dev/null
+++ b/roles/upload-logs-base/library/test-fixtures/.gitattributes
@@ -0,0 +1 @@
+*.* eol=lf
\ No newline at end of file
diff --git a/roles/upload-logs-base/library/test-fixtures/logs/Ꮓບບξ-unicode.txt b/roles/upload-logs-base/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt
similarity index 100%
rename from roles/upload-logs-base/library/test-fixtures/logs/Ꮓບບξ-unicode.txt
rename to roles/upload-logs-base/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt
diff --git a/roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo::3.txt b/roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt
similarity index 100%
rename from roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo::3.txt
rename to roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt
diff --git a/roles/upload-logs-base/library/test_index.py b/roles/upload-logs-base/library/test_index.py
index 69832d473..40df94777 100644
--- a/roles/upload-logs-base/library/test_index.py
+++ b/roles/upload-logs-base/library/test_index.py
@@ -33,6 +33,7 @@ import requests
from bs4 import BeautifulSoup
from .zuul_swift_upload import Uploader
from ..module_utils.zuul_jobs.upload_utils import FileList, Indexer, FileDetail
+from .filefixture import FileFixture
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
'test-fixtures')
@@ -50,8 +51,11 @@ class SymlinkFixture(fixtures.Fixture):
]
def _setUp(self):
+ self.file_fixture = FileFixture()
+ self.file_fixture.setUp()
+ self.addCleanup(self.file_fixture.cleanUp)
for (src, target) in self.links:
- path = os.path.join(FIXTURE_DIR, 'links', src)
+ path = os.path.join(self.file_fixture.root, 'links', src)
os.symlink(target, path)
self.addCleanup(os.unlink, path)
@@ -88,66 +92,69 @@ class TestFileList(testtools.TestCase):
'''Test a single directory with a trailing slash'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('controller', 'application/directory', None),
- ('zuul-info', 'application/directory', None),
- ('job-output.json', 'application/json', None),
- (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
- 'text/plain', None),
- ('controller/subdir', 'application/directory', None),
- ('controller/compressed.gz', 'text/plain', 'gzip'),
- ('controller/cpu-load.svg', 'image/svg+xml', None),
- ('controller/journal.xz', 'text/plain', 'xz'),
- ('controller/service_log.txt', 'text/plain', None),
- ('controller/syslog', 'text/plain', None),
- ('controller/subdir/foo::3.txt', 'text/plain', None),
- ('controller/subdir/subdir.txt', 'text/plain', None),
- ('zuul-info/inventory.yaml', 'text/plain', None),
- ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
- ])
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root, 'logs/'))
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('controller', 'application/directory', None),
+ ('zuul-info', 'application/directory', None),
+ ('job-output.json', 'application/json', None),
+ (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
+ 'text/plain', None),
+ ('controller/subdir', 'application/directory', None),
+ ('controller/compressed.gz', 'text/plain', 'gzip'),
+ ('controller/cpu-load.svg', 'image/svg+xml', None),
+ ('controller/journal.xz', 'text/plain', 'xz'),
+ ('controller/service_log.txt', 'text/plain', None),
+ ('controller/syslog', 'text/plain', None),
+ ('controller/subdir/foo::3.txt', 'text/plain', None),
+ ('controller/subdir/subdir.txt', 'text/plain', None),
+ ('zuul-info/inventory.yaml', 'text/plain', None),
+ ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+ ])
def test_single_dir(self):
'''Test a single directory without a trailing slash'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs'))
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('logs', 'application/directory', None),
- ('logs/controller', 'application/directory', None),
- ('logs/zuul-info', 'application/directory', None),
- ('logs/job-output.json', 'application/json', None),
- (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
- 'text/plain', None),
- ('logs/controller/subdir', 'application/directory', None),
- ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
- ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
- ('logs/controller/journal.xz', 'text/plain', 'xz'),
- ('logs/controller/service_log.txt', 'text/plain', None),
- ('logs/controller/syslog', 'text/plain', None),
- ('logs/controller/subdir/foo::3.txt', 'text/plain', None),
- ('logs/controller/subdir/subdir.txt', 'text/plain', None),
- ('logs/zuul-info/inventory.yaml', 'text/plain', None),
- ('logs/zuul-info/zuul-info.controller.txt',
- 'text/plain', None),
- ])
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root, 'logs'))
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('logs', 'application/directory', None),
+ ('logs/controller', 'application/directory', None),
+ ('logs/zuul-info', 'application/directory', None),
+ ('logs/job-output.json', 'application/json', None),
+ (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
+ 'text/plain', None),
+ ('logs/controller/subdir', 'application/directory', None),
+ ('logs/controller/compressed.gz', 'text/plain', 'gzip'),
+ ('logs/controller/cpu-load.svg', 'image/svg+xml', None),
+ ('logs/controller/journal.xz', 'text/plain', 'xz'),
+ ('logs/controller/service_log.txt', 'text/plain', None),
+ ('logs/controller/syslog', 'text/plain', None),
+ ('logs/controller/subdir/foo::3.txt', 'text/plain', None),
+ ('logs/controller/subdir/subdir.txt', 'text/plain', None),
+ ('logs/zuul-info/inventory.yaml', 'text/plain', None),
+ ('logs/zuul-info/zuul-info.controller.txt',
+ 'text/plain', None),
+ ])
def test_single_file(self):
'''Test a single file'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR,
- 'logs/zuul-info/inventory.yaml'))
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('inventory.yaml', 'text/plain', None),
- ])
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root,
+ 'logs/zuul-info/inventory.yaml'))
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('inventory.yaml', 'text/plain', None),
+ ])
def test_symlinks(self):
'''Test symlinks'''
with FileList() as fl:
- self.useFixture(SymlinkFixture())
- fl.add(os.path.join(FIXTURE_DIR, 'links/'))
+ symlink_fixture = self.useFixture(SymlinkFixture())
+ fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/'))
self.assert_files(fl, [
('', 'application/directory', None),
('controller', 'application/directory', None),
@@ -165,7 +172,8 @@ class TestFileList(testtools.TestCase):
def test_index_files(self):
'''Test index generation'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs'))
+ symlink_fixture = self.useFixture(SymlinkFixture())
+ fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs'))
ix = Indexer(fl)
ix.make_indexes()
@@ -223,32 +231,33 @@ class TestFileList(testtools.TestCase):
def test_index_files_trailing_slash(self):
'''Test index generation with a trailing slash'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
- ix = Indexer(fl)
- ix.make_indexes()
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root, 'logs/'))
+ ix = Indexer(fl)
+ ix.make_indexes()
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('controller', 'application/directory', None),
- ('zuul-info', 'application/directory', None),
- ('job-output.json', 'application/json', None),
- (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
- 'text/plain', None),
- ('index.html', 'text/html', None),
- ('controller/subdir', 'application/directory', None),
- ('controller/compressed.gz', 'text/plain', 'gzip'),
- ('controller/cpu-load.svg', 'image/svg+xml', None),
- ('controller/journal.xz', 'text/plain', 'xz'),
- ('controller/service_log.txt', 'text/plain', None),
- ('controller/syslog', 'text/plain', None),
- ('controller/index.html', 'text/html', None),
- ('controller/subdir/foo::3.txt', 'text/plain', None),
- ('controller/subdir/subdir.txt', 'text/plain', None),
- ('controller/subdir/index.html', 'text/html', None),
- ('zuul-info/inventory.yaml', 'text/plain', None),
- ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
- ('zuul-info/index.html', 'text/html', None),
- ])
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('controller', 'application/directory', None),
+ ('zuul-info', 'application/directory', None),
+ ('job-output.json', 'application/json', None),
+ (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
+ 'text/plain', None),
+ ('index.html', 'text/html', None),
+ ('controller/subdir', 'application/directory', None),
+ ('controller/compressed.gz', 'text/plain', 'gzip'),
+ ('controller/cpu-load.svg', 'image/svg+xml', None),
+ ('controller/journal.xz', 'text/plain', 'xz'),
+ ('controller/service_log.txt', 'text/plain', None),
+ ('controller/syslog', 'text/plain', None),
+ ('controller/index.html', 'text/html', None),
+ ('controller/subdir/foo::3.txt', 'text/plain', None),
+ ('controller/subdir/subdir.txt', 'text/plain', None),
+ ('controller/subdir/index.html', 'text/html', None),
+ ('zuul-info/inventory.yaml', 'text/plain', None),
+ ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+ ('zuul-info/index.html', 'text/html', None),
+ ])
top_index = self.find_file(fl, 'index.html')
page = open(top_index.full_path).read()
@@ -282,141 +291,154 @@ class TestFileList(testtools.TestCase):
def test_topdir_parent_link(self):
'''Test index generation creates topdir parent link'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
- ix = Indexer(fl)
- ix.make_indexes(
- create_parent_links=True,
- create_topdir_parent_link=True)
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root, 'logs/'))
+ ix = Indexer(fl)
+ ix.make_indexes(
+ create_parent_links=True,
+ create_topdir_parent_link=True)
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('controller', 'application/directory', None),
- ('zuul-info', 'application/directory', None),
- ('job-output.json', 'application/json', None),
- (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
- 'text/plain', None),
- ('index.html', 'text/html', None),
- ('controller/subdir', 'application/directory', None),
- ('controller/compressed.gz', 'text/plain', 'gzip'),
- ('controller/cpu-load.svg', 'image/svg+xml', None),
- ('controller/journal.xz', 'text/plain', 'xz'),
- ('controller/service_log.txt', 'text/plain', None),
- ('controller/syslog', 'text/plain', None),
- ('controller/index.html', 'text/html', None),
- ('controller/subdir/foo::3.txt', 'text/plain', None),
- ('controller/subdir/subdir.txt', 'text/plain', None),
- ('controller/subdir/index.html', 'text/html', None),
- ('zuul-info/inventory.yaml', 'text/plain', None),
- ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
- ('zuul-info/index.html', 'text/html', None),
- ])
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('controller', 'application/directory', None),
+ ('zuul-info', 'application/directory', None),
+ ('job-output.json', 'application/json', None),
+ (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
+ 'text/plain', None),
+ ('index.html', 'text/html', None),
+ ('controller/subdir', 'application/directory', None),
+ ('controller/compressed.gz', 'text/plain', 'gzip'),
+ ('controller/cpu-load.svg', 'image/svg+xml', None),
+ ('controller/journal.xz', 'text/plain', 'xz'),
+ ('controller/service_log.txt', 'text/plain', None),
+ ('controller/syslog', 'text/plain', None),
+ ('controller/index.html', 'text/html', None),
+ ('controller/subdir/foo::3.txt', 'text/plain', None),
+ ('controller/subdir/subdir.txt', 'text/plain', None),
+ ('controller/subdir/index.html', 'text/html', None),
+ ('zuul-info/inventory.yaml', 'text/plain', None),
+ ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+ ('zuul-info/index.html', 'text/html', None),
+ ])
- top_index = self.find_file(fl, 'index.html')
- page = open(top_index.full_path).read()
- page = BeautifulSoup(page, 'html.parser')
- rows = page.find_all('tr')[1:]
+ top_index = self.find_file(fl, 'index.html')
+ page = open(top_index.full_path).read()
+ page = BeautifulSoup(page, 'html.parser')
+ rows = page.find_all('tr')[1:]
- self.assertEqual(len(rows), 5)
+ self.assertEqual(len(rows), 5)
- self.assertEqual(rows[0].find('a').get('href'),
- '../index.html')
- self.assertEqual(rows[0].find('a').text, '../')
+ self.assertEqual(rows[0].find('a').get('href'),
+ '../index.html')
+ self.assertEqual(rows[0].find('a').text, '../')
- self.assertEqual(rows[1].find('a').get('href'),
- 'controller/index.html')
- self.assertEqual(rows[1].find('a').text, 'controller/')
+ self.assertEqual(rows[1].find('a').get('href'),
+ 'controller/index.html')
+ self.assertEqual(rows[1].find('a').text, 'controller/')
- self.assertEqual(rows[2].find('a').get('href'),
- 'zuul-info/index.html')
- self.assertEqual(rows[2].find('a').text, 'zuul-info/')
+ self.assertEqual(rows[2].find('a').get('href'),
+ 'zuul-info/index.html')
+ self.assertEqual(rows[2].find('a').text, 'zuul-info/')
- subdir_index = self.find_file(fl, 'controller/subdir/index.html')
- page = open(subdir_index.full_path).read()
- page = BeautifulSoup(page, 'html.parser')
- rows = page.find_all('tr')[1:]
- self.assertEqual(rows[0].find('a').get('href'), '../index.html')
- self.assertEqual(rows[0].find('a').text, '../')
+ subdir_index = self.find_file(
+ fl, 'controller/subdir/index.html'
+ )
+ page = open(subdir_index.full_path).read()
+ page = BeautifulSoup(page, 'html.parser')
+ rows = page.find_all('tr')[1:]
+ self.assertEqual(
+ rows[0].find('a').get('href'), '../index.html'
+ )
+ self.assertEqual(rows[0].find('a').text, '../')
- # Test proper escaping of files with funny names
- self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
- self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
- # Test files without escaping
- self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
- self.assertEqual(rows[2].find('a').text, 'subdir.txt')
+ # Test proper escaping of files with funny names
+ self.assertEqual(
+ rows[1].find('a').get('href'), 'foo%3A%3A3.txt'
+ )
+ self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
+ # Test files without escaping
+ self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
+ self.assertEqual(rows[2].find('a').text, 'subdir.txt')
def test_no_parent_links(self):
'''Test index generation creates topdir parent link'''
with FileList() as fl:
- fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
- ix = Indexer(fl)
- ix.make_indexes(
- create_parent_links=False,
- create_topdir_parent_link=False)
+ with FileFixture() as file_fixture:
+ fl.add(os.path.join(file_fixture.root, 'logs/'))
+ ix = Indexer(fl)
+ ix.make_indexes(
+ create_parent_links=False,
+ create_topdir_parent_link=False)
- self.assert_files(fl, [
- ('', 'application/directory', None),
- ('controller', 'application/directory', None),
- ('zuul-info', 'application/directory', None),
- ('job-output.json', 'application/json', None),
- (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
- 'text/plain', None),
- ('index.html', 'text/html', None),
- ('controller/subdir', 'application/directory', None),
- ('controller/compressed.gz', 'text/plain', 'gzip'),
- ('controller/cpu-load.svg', 'image/svg+xml', None),
- ('controller/journal.xz', 'text/plain', 'xz'),
- ('controller/service_log.txt', 'text/plain', None),
- ('controller/syslog', 'text/plain', None),
- ('controller/index.html', 'text/html', None),
- ('controller/subdir/foo::3.txt', 'text/plain', None),
- ('controller/subdir/subdir.txt', 'text/plain', None),
- ('controller/subdir/index.html', 'text/html', None),
- ('zuul-info/inventory.yaml', 'text/plain', None),
- ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
- ('zuul-info/index.html', 'text/html', None),
- ])
+ self.assert_files(fl, [
+ ('', 'application/directory', None),
+ ('controller', 'application/directory', None),
+ ('zuul-info', 'application/directory', None),
+ ('job-output.json', 'application/json', None),
+ (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
+ 'text/plain', None),
+ ('index.html', 'text/html', None),
+ ('controller/subdir', 'application/directory', None),
+ ('controller/compressed.gz', 'text/plain', 'gzip'),
+ ('controller/cpu-load.svg', 'image/svg+xml', None),
+ ('controller/journal.xz', 'text/plain', 'xz'),
+ ('controller/service_log.txt', 'text/plain', None),
+ ('controller/syslog', 'text/plain', None),
+ ('controller/index.html', 'text/html', None),
+ ('controller/subdir/foo::3.txt', 'text/plain', None),
+ ('controller/subdir/subdir.txt', 'text/plain', None),
+ ('controller/subdir/index.html', 'text/html', None),
+ ('zuul-info/inventory.yaml', 'text/plain', None),
+ ('zuul-info/zuul-info.controller.txt', 'text/plain', None),
+ ('zuul-info/index.html', 'text/html', None),
+ ])
- top_index = self.find_file(fl, 'index.html')
- page = open(top_index.full_path).read()
- page = BeautifulSoup(page, 'html.parser')
- rows = page.find_all('tr')[1:]
+ top_index = self.find_file(fl, 'index.html')
+ page = open(top_index.full_path).read()
+ page = BeautifulSoup(page, 'html.parser')
+ rows = page.find_all('tr')[1:]
- self.assertEqual(len(rows), 4)
+ self.assertEqual(len(rows), 4)
- self.assertEqual(rows[0].find('a').get('href'),
- 'controller/index.html')
- self.assertEqual(rows[0].find('a').text,
- 'controller/')
+ self.assertEqual(rows[0].find('a').get('href'),
+ 'controller/index.html')
+ self.assertEqual(rows[0].find('a').text,
+ 'controller/')
- self.assertEqual(rows[1].find('a').get('href'),
- 'zuul-info/index.html')
- self.assertEqual(rows[1].find('a').text,
- 'zuul-info/')
+ self.assertEqual(rows[1].find('a').get('href'),
+ 'zuul-info/index.html')
+ self.assertEqual(rows[1].find('a').text,
+ 'zuul-info/')
- subdir_index = self.find_file(fl, 'controller/subdir/index.html')
- page = open(subdir_index.full_path).read()
- page = BeautifulSoup(page, 'html.parser')
- rows = page.find_all('tr')[1:]
+ subdir_index = self.find_file(
+ fl, 'controller/subdir/index.html'
+ )
+ page = open(subdir_index.full_path).read()
+ page = BeautifulSoup(page, 'html.parser')
+ rows = page.find_all('tr')[1:]
- # Test proper escaping of files with funny names
- self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt')
- self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
- # Test files without escaping
- self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
- self.assertEqual(rows[1].find('a').text, 'subdir.txt')
+ # Test proper escaping of files with funny names
+ self.assertEqual(
+ rows[0].find('a').get('href'), 'foo%3A%3A3.txt'
+ )
+ self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
+ # Test files without escaping
+ self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
+ self.assertEqual(rows[1].find('a').text, 'subdir.txt')
class TestFileDetail(testtools.TestCase):
def test_get_file_detail(self):
'''Test files info'''
- path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
- file_detail = FileDetail(path, '')
- path_stat = os.stat(path)
- self.assertEqual(
- time.gmtime(path_stat[stat.ST_MTIME]),
- file_detail.last_modified)
- self.assertEqual(16, file_detail.size)
+ with FileFixture() as file_fixture:
+ path = os.path.join(file_fixture.root, 'logs/job-output.json')
+ file_detail = FileDetail(path, '')
+ path_stat = os.stat(path)
+ self.assertEqual(
+ time.gmtime(path_stat[stat.ST_MTIME]),
+ file_detail.last_modified)
+ self.assertEqual(16, file_detail.size)
def test_get_file_detail_missing_file(self):
'''Test files that go missing during a walk'''
@@ -456,26 +478,29 @@ class TestUpload(testtools.TestCase):
)
# Get some test files to upload
- files = [
- FileDetail(
- os.path.join(FIXTURE_DIR, "logs/job-output.json"),
- "job-output.json",
- ),
- FileDetail(
- os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"),
- "inventory.yaml",
- ),
- ]
-
- expected_failures = [
- {
- "file": "job-output.json",
- "error": (
- "Error posting file after multiple attempts: "
- "Failed for a reason"
+ with FileFixture() as file_fixture:
+ files = [
+ FileDetail(
+ os.path.join(file_fixture.root, "logs/job-output.json"),
+ "job-output.json",
),
- },
- ]
+ FileDetail(
+ os.path.join(
+ file_fixture.root, "logs/zuul-info/inventory.yaml"
+ ),
+ "inventory.yaml",
+ ),
+ ]
- failures = uploader.upload(files)
- self.assertEqual(expected_failures, failures)
+ expected_failures = [
+ {
+ "file": "job-output.json",
+ "error": (
+ "Error posting file after multiple attempts: "
+ "Failed for a reason"
+ ),
+ },
+ ]
+
+ failures = uploader.upload(files)
+ self.assertEqual(expected_failures, failures)