Merge "[results_uploader] Use pathlib for all file-related operations" into main
diff --git a/tools/results_uploader/src/mobly_result_converter.py b/tools/results_uploader/src/mobly_result_converter.py
index 9348add..fab584b 100644
--- a/tools/results_uploader/src/mobly_result_converter.py
+++ b/tools/results_uploader/src/mobly_result_converter.py
@@ -36,9 +36,7 @@
import dataclasses
import datetime
import enum
-import glob
import logging
-import os
import pathlib
import re
from typing import Any, Dict, Iterator, List, Mapping, Optional
@@ -60,11 +58,6 @@
_ILLEGAL_YAML_CHARS = re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]')
-def as_posix_path(path):
- """Returns a given path as a string in POSIX format."""
- return str(pathlib.Path(path).as_posix())
-
-
class MoblyResultstoreProperties(enum.Enum):
"""Resultstore properties defined specifically for all Mobly tests.
@@ -204,54 +197,45 @@
def _add_file_annotations(
entry: Mapping[str, Any],
properties_element: ElementTree.Element,
- mobly_base_directory: Optional[str],
- resultstore_root_directory: Optional[str],
+ mobly_base_directory: Optional[pathlib.Path],
) -> None:
"""Adds file annotations for a Mobly test case files.
The mobly_base_directory is used to find the files belonging to a test case.
The files under "mobly_base_directory/test_class/test_method" belong to the
- test_class#test_method Resultstore node.
-
- The resultstore_root_directory is used to determine the
- relative path of the files to the Resultstore root directory for undeclared
- outputs. The file annotation must be written for the relative path.
+ test_class#test_method Resultstore node. Additionally, it is used to
+ determine the relative path of the files for Resultstore undeclared outputs.
+ The file annotation must be written for the relative path.
Args:
entry: Mobly summary entry for the test case.
properties_element: Test case properties element.
mobly_base_directory: Base directory of the Mobly test.
- resultstore_root_directory: Root directory for Resultstore undeclared
- outputs.
"""
- # If these directories are not provided, the converter will not add the
+ # If mobly_base_directory is not provided, the converter will not add the
# annotations to associate the files with the test cases.
if (
mobly_base_directory is None
- or resultstore_root_directory is None
or entry.get(records.TestResultEnums.RECORD_SIGNATURE, None) is None
):
return
test_class = entry[records.TestResultEnums.RECORD_CLASS]
- test_case_directory = os.path.join(
- mobly_base_directory,
+ test_case_directory = mobly_base_directory.joinpath(
test_class,
- entry[records.TestResultEnums.RECORD_SIGNATURE],
+ entry[records.TestResultEnums.RECORD_SIGNATURE]
)
- test_case_files = glob.glob(
- os.path.join(test_case_directory, '**'), recursive=True
- )
+ test_case_files = test_case_directory.rglob('*')
file_counter = 0
for file_path in test_case_files:
- if not os.path.isfile(file_path):
+ if not file_path.is_file():
continue
- relative_path = os.path.relpath(file_path, resultstore_root_directory)
+ relative_path = file_path.relative_to(mobly_base_directory)
_add_or_update_property_element(
properties_element,
f'test_output{file_counter}',
- as_posix_path(relative_path),
+ str(relative_path.as_posix()),
)
file_counter += 1
@@ -447,8 +431,7 @@
def _process_record(
entry: Mapping[str, Any],
reran_node: Optional[ReranNode],
- mobly_base_directory: Optional[str],
- resultstore_root_directory: Optional[str],
+ mobly_base_directory: Optional[pathlib.Path],
) -> ElementTree.Element:
"""Processes a single Mobly test record entry to a Resultstore test case
node.
@@ -459,8 +442,6 @@
if this test is part of a rerun chain.
mobly_base_directory: Base directory for the Mobly test. Artifacts from
the Mobly test will be saved here.
- resultstore_root_directory: Root directory for Resultstore undeclared
- outputs.
Returns:
A Resultstore XML node representing a single test case.
@@ -577,7 +558,6 @@
entry,
properties_element,
mobly_base_directory,
- resultstore_root_directory,
)
if entry[records.TestResultEnums.RECORD_UID] is not None:
@@ -655,28 +635,25 @@
def convert(
- mobly_results_path: str,
- mobly_base_directory: Optional[str] = None,
- resultstore_root_directory: Optional[str] = None,
+ mobly_results_path: pathlib.Path,
+ mobly_base_directory: Optional[pathlib.Path] = None,
) -> ElementTree.ElementTree:
"""Converts a Mobly results summary file to Resultstore XML schema.
- The mobly_base_directory and resultstore_root_directory will be used to
- compute the file links for each Resultstore tree element. If these are
- absent then the file links will be omitted.
+ The mobly_base_directory will be used to compute the file links for each
+ Resultstore tree element. If it is absent then the file links will be
+ omitted.
Args:
mobly_results_path: Path to the Mobly summary YAML file.
mobly_base_directory: Base directory of the Mobly test.
- resultstore_root_directory: Root directory for Resultstore undeclared
- outputs.
Returns:
A Resultstore XML tree for the Mobly test.
"""
logging.info('Generating Resultstore tree...')
- with open(mobly_results_path, 'r', encoding='utf-8') as f:
+ with mobly_results_path.open('r', encoding='utf-8') as f:
summary_entries = list(
yaml.safe_load_all(_ILLEGAL_YAML_CHARS.sub('', f.read()))
)
@@ -694,21 +671,16 @@
mobly_root_properties = _create_or_return_properties_element(
mobly_test_root)
# Add files under the Mobly root directory to the Mobly test suite node.
- if (
- mobly_base_directory is not None
- and resultstore_root_directory is not None
- ):
+ if mobly_base_directory is not None:
file_counter = 0
- for filename in os.listdir(mobly_base_directory):
- file_path = os.path.join(mobly_base_directory, filename)
- if not os.path.isfile(file_path):
+ for file_path in mobly_base_directory.iterdir():
+ if not file_path.is_file():
continue
- relative_path = os.path.relpath(file_path,
- resultstore_root_directory)
+ relative_path = file_path.relative_to(mobly_base_directory)
_add_or_update_property_element(
mobly_root_properties,
f'test_output{file_counter}',
- as_posix_path(relative_path),
+ str(relative_path.as_posix()),
)
file_counter += 1
@@ -760,10 +732,7 @@
else:
reran_node = None
class_elements[class_name].append(
- _process_record(
- entry, reran_node, mobly_base_directory,
- resultstore_root_directory
- )
+ _process_record(entry, reran_node, mobly_base_directory)
)
user_data_entries = [
diff --git a/tools/results_uploader/src/results_uploader.py b/tools/results_uploader/src/results_uploader.py
index 08ada2a..6a949c9 100644
--- a/tools/results_uploader/src/results_uploader.py
+++ b/tools/results_uploader/src/results_uploader.py
@@ -22,7 +22,6 @@
from importlib import resources
import logging
import mimetypes
-import os
import pathlib
import platform
import shutil
@@ -84,33 +83,32 @@
target_id: str | None = None
-def _convert_results(mobly_dir: str, dest_dir: str) -> _TestResultInfo:
- """Converts Mobly test results into a Resultstore artifacts."""
+def _convert_results(
+ mobly_dir: pathlib.Path, dest_dir: pathlib.Path) -> _TestResultInfo:
+ """Converts Mobly test results into Resultstore artifacts."""
test_result_info = _TestResultInfo()
logging.info('Converting raw Mobly logs into Resultstore artifacts...')
# Generate the test.xml
- mobly_yaml_path = os.path.join(mobly_dir, _TEST_SUMMARY_YAML)
- if os.path.isfile(mobly_yaml_path):
- test_xml = mobly_result_converter.convert(
- mobly_yaml_path, mobly_dir, mobly_dir
- )
+ mobly_yaml_path = mobly_dir.joinpath(_TEST_SUMMARY_YAML)
+ if mobly_yaml_path.is_file():
+ test_xml = mobly_result_converter.convert(mobly_yaml_path, mobly_dir)
ElementTree.indent(test_xml)
test_xml.write(
- os.path.join(dest_dir, _TEST_XML),
+ str(dest_dir.joinpath(_TEST_XML)),
encoding='utf-8',
xml_declaration=True,
)
test_result_info = _get_test_result_info_from_test_xml(test_xml)
# Copy test_log.INFO to test.log
- test_log_info = os.path.join(mobly_dir, _TEST_LOG_INFO)
- if os.path.isfile(test_log_info):
- shutil.copyfile(test_log_info, os.path.join(dest_dir, _TEST_LOGS))
+ test_log_info = mobly_dir.joinpath(_TEST_LOG_INFO)
+ if test_log_info.is_file():
+ shutil.copyfile(test_log_info, dest_dir.joinpath(_TEST_LOGS))
# Copy directory to undeclared_outputs/
shutil.copytree(
mobly_dir,
- os.path.join(dest_dir, _UNDECLARED_OUTPUTS),
+ dest_dir.joinpath(_UNDECLARED_OUTPUTS),
dirs_exist_ok=True,
)
return test_result_info
@@ -186,7 +184,7 @@
def _upload_dir_to_gcs(
- src_dir: str, gcs_bucket: str, gcs_dir: str
+ src_dir: pathlib.Path, gcs_bucket: str, gcs_dir: str
) -> list[str]:
"""Uploads the given directory to a GCS bucket."""
# Set correct MIME types for certain text-format files.
@@ -196,7 +194,7 @@
bucket_obj = storage.Client().bucket(gcs_bucket)
- glob = pathlib.Path(src_dir).expanduser().rglob('*')
+ glob = src_dir.rglob('*')
file_paths = [
str(path.relative_to(src_dir).as_posix())
for path in glob
@@ -204,9 +202,9 @@
]
logging.info(
- 'Uploading %s files from %s to Cloud Storage bucket %s/%s...',
+ 'Uploading %s files from %s to Cloud Storage directory %s/%s...',
len(file_paths),
- src_dir,
+ str(src_dir),
gcs_bucket,
gcs_dir,
)
@@ -224,7 +222,7 @@
results = transfer_manager.upload_many_from_filenames(
bucket_obj,
file_paths,
- source_directory=src_dir,
+ source_directory=str(src_dir),
blob_name_prefix=blob_name_prefix,
worker_type=worker_type,
)
@@ -246,7 +244,7 @@
return success_paths
-def _prompt_user_upload(src_dir: str, gcs_bucket: str) -> None:
+def _prompt_user_upload(src_dir: pathlib.Path, gcs_bucket: str) -> None:
"""Prompts the user to manually upload files to GCS."""
print(_GCS_UPLOAD_INSTRUCTIONS % (gcs_bucket, src_dir))
while True:
@@ -327,9 +325,10 @@
else args.gcs_dir
)
with tempfile.TemporaryDirectory() as tmp:
- converted_dir = os.path.join(tmp, gcs_dir)
- os.mkdir(converted_dir)
- test_result_info = _convert_results(args.mobly_dir, converted_dir)
+ converted_dir = pathlib.Path(tmp).joinpath(gcs_dir)
+ converted_dir.mkdir()
+ mobly_dir = pathlib.Path(args.mobly_dir).absolute().expanduser()
+ test_result_info = _convert_results(mobly_dir, converted_dir)
gcs_files = _upload_dir_to_gcs(
converted_dir, gcs_bucket, gcs_dir)
_upload_to_resultstore(
