{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Let's verify the downloaded files not broken\n\n* I'll use MD5 and SHA1 hasusum to verify downloaded files and Kaggle Notebook's files are same.\n* To use, you run this notebook in your local-machine, and check df_output.csv's hashsume are same in this notebook.\n* Check your dataset's filepath in your local-machine. **Please check the 4th cell.**"},{"metadata":{},"cell_type":"markdown","source":"## Calculate hashsum"},{"metadata":{"trusted":true},"cell_type":"code","source":"import hashlib\nimport os\nimport pathlib\nimport platform\nimport sys\nimport warnings\n\nimport pandas as pd\n\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Filter warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get working directory\ntry:\n    path_working_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))\nexcept:\n    path_working_dir = os.path.abspath(str(pathlib.Path().resolve()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Change the next cell in your local-machine"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set input directory\npath_input  = '../input/rsna-intracranial-hemorrhage-detection'\npath_output = './'\nassert os.path.exists(path_input) == True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calc hashsum of files with selected file extension\nlist_output = []\nlist_ext    = ['.csv', '.dcm']\nfor (dirpath, dirnames, filenames) in os.walk(path_input):\n    print(dirpath.replace(path_input, '.'), file=sys.stderr)\n    for filename in tqdm(filenames):\n        if max([filename.find(ext) for ext in list_ext]) > -1:\n            with open(os.path.join(dirpath, filename), 'rb') as fp:\n                fp_read = fp.read()\n                list_output.append([os.path.join(dirpath.replace(path_input, '.'), filename), hashlib.md5(fp_read).hexdigest(), hashlib.sha1(fp_read).hexdigest()])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Output result\ndf_output = pd.DataFrame(list_output, columns=['filename', 'md5', 'sha1'])\ndf_output = df_output.sort_values(by='filename')\ndf_output.to_csv(os.path.join(path_output, 'df_output.csv'), index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_output.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check df_output.csv's hashsum\n* `md5sum` and `sha1sum` commands can use in Ubuntu\n* `md5` and `openssl sha1` commands can use in OS X\n* `certutil -hashfile df_output.csv MD5` and `certutil -hashfile df_output.csv SHA1` commands can use in Windows 7-10"},{"metadata":{"trusted":true},"cell_type":"code","source":"!md5sum df_output.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!sha1sum df_output.csv","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Let's enjoy this competition! 🎉"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}