Separate targets and snapshot/timestamp schemas

This separation and refactoring is part of the change to
make length and hashes optional for timestamp and snapshot roles.

It separates FILEINFO_SCHEMA into two separate schemas:
TARGETS_FILEINFO_SCHEMA and METADATA_FILEINFO_SCHEMA.
The distinction is needed because as of version 1.0.1 of the tuf
spec targets role has mandatory length and hashes, and
snapshot and timestamp roles have a mandatory version, and optional
length and hashes.
That's why targets can't share the same schemas
as timestamp and snapshot.

Because of that schema distinction, make_fileinfo had to be too
separated into make_targets_fileinfo and make_metadata_fileinfo.

Signed-off-by: Martin Vrachev <mvrachev@vmware.com>

Author: Martin Vrachev

tests/test_arbitrary_package_attack.py

@@ -180,7 +180,7 @@ def test_without_tuf(self):
     client_target_path = os.path.join(self.client_directory, 'file1.txt')
     self.assertFalse(os.path.exists(client_target_path))
     length, hashes = securesystemslib.util.get_file_details(target_path)
-    fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     url_prefix = self.repository_mirrors['mirror1']['url_prefix']
     url_file = os.path.join(url_prefix, 'targets', 'file1.txt')
@@ -190,20 +190,20 @@ def test_without_tuf(self):
 
     self.assertTrue(os.path.exists(client_target_path))
     length, hashes = securesystemslib.util.get_file_details(client_target_path)
-    download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
     self.assertEqual(fileinfo, download_fileinfo)
 
     # Test: Download a target file that has been modified by an attacker.
     with open(target_path, 'wt') as file_object:
       file_object.write('add malicious content.')
     length, hashes = securesystemslib.util.get_file_details(target_path)
-    malicious_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    malicious_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     # On Windows, the URL portion should not contain back slashes.
     six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_target_path)
 
     length, hashes = securesystemslib.util.get_file_details(client_target_path)
-    download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     # Verify 'download_fileinfo' is unequal to the original trusted version.
     self.assertNotEqual(download_fileinfo, fileinfo)

tests/test_endless_data_attack.py

@@ -184,7 +184,7 @@ def test_without_tuf(self):
     client_target_path = os.path.join(self.client_directory, 'file1.txt')
     self.assertFalse(os.path.exists(client_target_path))
     length, hashes = securesystemslib.util.get_file_details(target_path)
-    fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     url_prefix = self.repository_mirrors['mirror1']['url_prefix']
     url_file = os.path.join(url_prefix, 'targets', 'file1.txt')
@@ -194,15 +194,15 @@ def test_without_tuf(self):
 
     self.assertTrue(os.path.exists(client_target_path))
     length, hashes = securesystemslib.util.get_file_details(client_target_path)
-    download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
     self.assertEqual(fileinfo, download_fileinfo)
 
     # Test: Download a target file that has been modified by an attacker with
     # extra data.
     with open(target_path, 'a') as file_object:
       file_object.write('append large amount of data' * 100000)
     large_length, hashes = securesystemslib.util.get_file_details(target_path)
-    malicious_fileinfo = tuf.formats.make_fileinfo(large_length, hashes)
+    malicious_fileinfo = tuf.formats.make_targets_fileinfo(large_length, hashes)
 
     # Is the modified file actually larger?
     self.assertTrue(large_length > length)
@@ -211,7 +211,7 @@ def test_without_tuf(self):
     six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_target_path)
 
     length, hashes = securesystemslib.util.get_file_details(client_target_path)
-    download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     # Verify 'download_fileinfo' is unequal to the original trusted version.
     self.assertNotEqual(download_fileinfo, fileinfo)
@@ -235,10 +235,10 @@ def test_with_tuf(self):
     # Verify the client's downloaded file matches the repository's.
     target_path = os.path.join(self.repository_directory, 'targets', 'file1.txt')
     length, hashes = securesystemslib.util.get_file_details(client_target_path)
-    fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     length, hashes = securesystemslib.util.get_file_details(client_target_path)
-    download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
     self.assertEqual(fileinfo, download_fileinfo)
 
     # Modify 'file1.txt' and confirm that the TUF client only downloads up to
@@ -257,7 +257,7 @@ def test_with_tuf(self):
     # extra data appended should be discarded by the client, so the downloaded
     # file size and hash should not have changed.
     length, hashes = securesystemslib.util.get_file_details(client_target_path)
-    download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
     self.assertEqual(fileinfo, download_fileinfo)
 
     # Test that the TUF client does not download large metadata files, as well.

tests/test_formats.py

@@ -119,11 +119,14 @@ def test_schemas(self):
                          'keyval': {'public': 'pubkey',
                                     'private': 'privkey'}}),
 
-      'FILEINFO_SCHEMA': (tuf.formats.FILEINFO_SCHEMA,
+      'TARGETS_FILEINFO_SCHEMA': (tuf.formats.TARGETS_FILEINFO_SCHEMA,
                           {'length': 1024,
                            'hashes': {'sha256': 'A4582BCF323BCEF'},
                            'custom': {'type': 'paintjob'}}),
 
+      'METADATA_FILEINFO_SCHEMA': (tuf.formats.METADATA_FILEINFO_SCHEMA,
+                          {'version': 1}),
+
       'FILEDICT_SCHEMA': (tuf.formats.FILEDICT_SCHEMA,
                           {'metadata/root.json': {'length': 1024,
                                                  'hashes': {'sha256': 'ABCD123'},
@@ -241,7 +244,8 @@ def test_schemas(self):
          'version': 8,
          'expires': '1985-10-21T13:20:00Z',
          'meta': {'metadattimestamp.json': {'length': 1024,
-                                            'hashes': {'sha256': 'AB1245'}}}}),
+                                            'hashes': {'sha256': 'AB1245'},
+                                            'version': 1}}}),
 
       'MIRROR_SCHEMA': (tuf.formats.MIRROR_SCHEMA,
         {'url_prefix': 'http://localhost:8001',
@@ -394,7 +398,7 @@ def test_build_dict_conforming_to_schema(self):
     length = 88
     hashes = {'sha256': '3c7fe3eeded4a34'}
     expires = '1985-10-21T13:20:00Z'
-    filedict = {'snapshot.json': {'length': length, 'hashes': hashes}}
+    filedict = {'snapshot.json': {'length': length, 'hashes': hashes, 'version': 1}}
 
 
     # Try with and without _type and spec_version, both of which are
@@ -797,28 +801,60 @@ def test_make_signable(self):
 
 
 
-  def test_make_fileinfo(self):
+  def test_make_targets_fileinfo(self):
     # Test conditions for valid arguments.
     length = 1024
     hashes = {'sha256': 'A4582BCF323BCEF', 'sha512': 'A4582BCF323BFEF'}
-    version = 8
     custom = {'type': 'paintjob'}
 
-    FILEINFO_SCHEMA = tuf.formats.FILEINFO_SCHEMA
-    make_fileinfo = tuf.formats.make_fileinfo
-    self.assertTrue(FILEINFO_SCHEMA.matches(make_fileinfo(length, hashes, version, custom)))
-    self.assertTrue(FILEINFO_SCHEMA.matches(make_fileinfo(length, hashes)))
+    TARGETS_FILEINFO_SCHEMA = tuf.formats.TARGETS_FILEINFO_SCHEMA
+    make_targets_fileinfo = tuf.formats.make_targets_fileinfo
+    self.assertTrue(TARGETS_FILEINFO_SCHEMA.matches(make_targets_fileinfo(length, hashes, custom)))
+    self.assertTrue(TARGETS_FILEINFO_SCHEMA.matches(make_targets_fileinfo(length, hashes)))
 
     # Test conditions for invalid arguments.
     bad_length = 'bad'
     bad_hashes = 'bad'
     bad_custom = 'bad'
 
-    self.assertRaises(securesystemslib.exceptions.FormatError, make_fileinfo, bad_length, hashes, custom)
-    self.assertRaises(securesystemslib.exceptions.FormatError, make_fileinfo, length, bad_hashes, custom)
-    self.assertRaises(securesystemslib.exceptions.FormatError, make_fileinfo, length, hashes, bad_custom)
-    self.assertRaises(securesystemslib.exceptions.FormatError, make_fileinfo, bad_length, hashes)
-    self.assertRaises(securesystemslib.exceptions.FormatError, make_fileinfo, length, bad_hashes)
+    self.assertRaises(securesystemslib.exceptions.FormatError, make_targets_fileinfo,
+      bad_length, hashes, custom)
+    self.assertRaises(securesystemslib.exceptions.FormatError, make_targets_fileinfo,
+      length, bad_hashes, custom)
+    self.assertRaises(securesystemslib.exceptions.FormatError, make_targets_fileinfo,
+      length, hashes, bad_custom)
+    self.assertRaises(securesystemslib.exceptions.FormatError, make_targets_fileinfo,
+      bad_length, hashes)
+    self.assertRaises(securesystemslib.exceptions.FormatError, make_targets_fileinfo,
+      length, bad_hashes)
+
+
+
+  def test_make_metadata_fileinfo(self):
+    # Test conditions for valid arguments.
+    length = 1024
+    hashes = {'sha256': 'A4582BCF323BCEF', 'sha512': 'A4582BCF323BFEF'}
+    version = 8
+
+    METADATA_FILEINFO_SCHEMA = tuf.formats.METADATA_FILEINFO_SCHEMA
+    make_metadata_fileinfo = tuf.formats.make_metadata_fileinfo
+    self.assertTrue(METADATA_FILEINFO_SCHEMA.matches(make_metadata_fileinfo(
+        version, length, hashes)))
+    self.assertTrue(METADATA_FILEINFO_SCHEMA.matches(make_metadata_fileinfo(version)))
+
+    # Test conditions for invalid arguments.
+    bad_version = 'bad'
+    bad_length = 'bad'
+    bad_hashes = 'bad'
+
+    self.assertRaises(securesystemslib.exceptions.FormatError, make_metadata_fileinfo,
+        bad_version, length, hashes)
+    self.assertRaises(securesystemslib.exceptions.FormatError, make_metadata_fileinfo,
+        version, bad_length, hashes)
+    self.assertRaises(securesystemslib.exceptions.FormatError, make_metadata_fileinfo,
+        version, length, bad_hashes)
+    self.assertRaises(securesystemslib.exceptions.FormatError, make_metadata_fileinfo,
+        bad_version)
 
 
 

tests/test_indefinite_freeze_attack.py

@@ -230,15 +230,15 @@ def test_without_tuf(self):
     shutil.copy(timestamp_path, client_timestamp_path)
 
     length, hashes = securesystemslib.util.get_file_details(timestamp_path)
-    fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     url_prefix = self.repository_mirrors['mirror1']['url_prefix']
     url_file = os.path.join(url_prefix, 'metadata', 'timestamp.json')
 
     six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path)
 
     length, hashes = securesystemslib.util.get_file_details(client_timestamp_path)
-    download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     # Verify 'download_fileinfo' is equal to the current local file.
     self.assertEqual(download_fileinfo, fileinfo)

tests/test_replay_attack.py

@@ -205,7 +205,7 @@ def test_without_tuf(self):
     # The fileinfo of the previous version is saved to verify that it is indeed
     # accepted by the non-TUF client.
     length, hashes = securesystemslib.util.get_file_details(backup_timestamp)
-    previous_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    previous_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     # Modify the timestamp file on the remote repository.
     repository = repo_tool.load_repository(self.repository_directory)
@@ -227,7 +227,7 @@ def test_without_tuf(self):
     # Save the fileinfo of the new version generated to verify that it is
     # saved by the client.
     length, hashes = securesystemslib.util.get_file_details(timestamp_path)
-    new_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    new_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     url_prefix = self.repository_mirrors['mirror1']['url_prefix']
     url_file = os.path.join(url_prefix, 'metadata', 'timestamp.json')
@@ -238,7 +238,7 @@ def test_without_tuf(self):
     six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path)
 
     length, hashes = securesystemslib.util.get_file_details(client_timestamp_path)
-    download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     # Verify 'download_fileinfo' is equal to the new version.
     self.assertEqual(download_fileinfo, new_fileinfo)
@@ -251,7 +251,7 @@ def test_without_tuf(self):
     six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path)
 
     length, hashes = securesystemslib.util.get_file_details(client_timestamp_path)
-    download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     # Verify 'download_fileinfo' is equal to the previous version.
     self.assertEqual(download_fileinfo, previous_fileinfo)
@@ -278,7 +278,7 @@ def test_with_tuf(self):
     # The fileinfo of the previous version is saved to verify that it is indeed
     # accepted by the non-TUF client.
     length, hashes = securesystemslib.util.get_file_details(backup_timestamp)
-    previous_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    previous_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     # Modify the timestamp file on the remote repository.
     repository = repo_tool.load_repository(self.repository_directory)
@@ -300,7 +300,7 @@ def test_with_tuf(self):
     # Save the fileinfo of the new version generated to verify that it is
     # saved by the client.
     length, hashes = securesystemslib.util.get_file_details(timestamp_path)
-    new_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    new_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     # Refresh top-level metadata, including 'timestamp.json'.  Installation of
     # new version of 'timestamp.json' is expected.
@@ -309,7 +309,7 @@ def test_with_tuf(self):
     client_timestamp_path = os.path.join(self.client_directory,
         self.repository_name, 'metadata', 'current', 'timestamp.json')
     length, hashes = securesystemslib.util.get_file_details(client_timestamp_path)
-    download_fileinfo = tuf.formats.make_fileinfo(length, hashes)
+    download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes)
 
     # Verify 'download_fileinfo' is equal to the new version.
     self.assertEqual(download_fileinfo, new_fileinfo)

tests/test_repository_lib.py

@@ -227,7 +227,7 @@ def test_get_metadata_filenames(self):
 
 
 
-  def test_get_metadata_fileinfo(self):
+  def test_get_targets_metadata_fileinfo(self):
     # Test normal case.
     temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory)
     test_filepath = os.path.join(temporary_directory, 'file.txt')
@@ -236,31 +236,31 @@ def test_get_metadata_fileinfo(self):
       file_object.write('test file')
 
     # Generate test fileinfo object.  It is assumed SHA256 and SHA512 hashes
-    # are computed by get_metadata_fileinfo().
+    # are computed by get_targets_metadata_fileinfo().
     file_length = os.path.getsize(test_filepath)
     sha256_digest_object = securesystemslib.hash.digest_filename(test_filepath)
     sha512_digest_object = securesystemslib.hash.digest_filename(test_filepath, algorithm='sha512')
     file_hashes = {'sha256': sha256_digest_object.hexdigest(),
                    'sha512': sha512_digest_object.hexdigest()}
     fileinfo = {'length': file_length, 'hashes': file_hashes}
-    self.assertTrue(tuf.formats.FILEINFO_SCHEMA.matches(fileinfo))
+    self.assertTrue(tuf.formats.TARGETS_FILEINFO_SCHEMA.matches(fileinfo))
 
     storage_backend = securesystemslib.storage.FilesystemBackend()
 
-    self.assertEqual(fileinfo, repo_lib.get_metadata_fileinfo(test_filepath,
+    self.assertEqual(fileinfo, repo_lib.get_targets_metadata_fileinfo(test_filepath,
                                                               storage_backend))
 
 
     # Test improperly formatted argument.
     self.assertRaises(securesystemslib.exceptions.FormatError,
-                      repo_lib.get_metadata_fileinfo, 3,
+                      repo_lib.get_targets_metadata_fileinfo, 3,
                       storage_backend)
 
 
     # Test non-existent file.
     nonexistent_filepath = os.path.join(temporary_directory, 'oops.txt')
     self.assertRaises(securesystemslib.exceptions.Error,
-                      repo_lib.get_metadata_fileinfo,
+                      repo_lib.get_targets_metadata_fileinfo,
                       nonexistent_filepath, storage_backend)
 
 
@@ -589,7 +589,7 @@ def test_generate_snapshot_metadata_without_hashes_and_length(self):
         if metadata_filename not in \
             ['root.json', 'targets.json', 'timestamp.json', 'snapshot.json']:
 
-          # Check that both length and hashes are not is not calculated
+          # Check that both length and hashes are not are not calculated
           self.assertNotIn('length', metadata_files_info_dict[metadata_filename])
           self.assertNotIn('hashes', metadata_files_info_dict[metadata_filename])
 

tests/test_repository_tool.py

@@ -460,8 +460,8 @@ def test_writeall_no_files(self):
     # Add target fileinfo
     target1_hashes = {'sha256': 'c2986576f5fdfd43944e2b19e775453b96748ec4fe2638a6d2f32f1310967095'}
     target2_hashes = {'sha256': '517c0ce943e7274a2431fa5751e17cfd5225accd23e479bfaad13007751e87ef'}
-    target1_fileinfo = tuf.formats.make_fileinfo(555, target1_hashes)
-    target2_fileinfo = tuf.formats.make_fileinfo(37, target2_hashes)
+    target1_fileinfo = tuf.formats.make_targets_fileinfo(555, target1_hashes)
+    target2_fileinfo = tuf.formats.make_targets_fileinfo(37, target2_hashes)
     target1 = 'file1.txt'
     target2 = 'file2.txt'
     repository.targets.add_target(target1, fileinfo=target1_fileinfo)
@@ -1527,7 +1527,7 @@ def test_add_target_to_bin(self):
 
     # Test adding a target with fileinfo
     target2_hashes = {'sha256': '517c0ce943e7274a2431fa5751e17cfd5225accd23e479bfaad13007751e87ef'}
-    target2_fileinfo = tuf.formats.make_fileinfo(37, target2_hashes)
+    target2_fileinfo = tuf.formats.make_targets_fileinfo(37, target2_hashes)
     target2_filepath = 'file2.txt'
 
     self.targets_object.add_target_to_bin(target2_filepath, 16, fileinfo=target2_fileinfo)