Related
I am currently trying to get the Caltech camera traps benchmark dataset into TFRecords but I am struggling quite a bit. https://lila.science/datasets/caltech-camera-traps. The annotations are displayed as follows:
"info": {"contributor": "Sara Beery", "date_created": "2018-07-03 18:34:36.573636", "version": "Caltech Camera Traps - ECCV18", "description": "Database of camera trap images collected from the NPS and the USGS with help from Justin Brown and Erin Boydston", "year": 2018}]
"categories": [{"id": 6, "name": "bobcat"}, ....]
"images": [{"file_name": "59b93afb-23d2-11e8-a6a3-ec086b02610b.jpg", "rights_holder": "Justin Brown", "height": 1494, "width": 2048, "frame_num": 2, "date_captured": "2012-05-09 07:33:45", "location": 38, "seq_num_frames": 3, "seq_id": "6f04895c-5567-11e8-a3d6-dca9047ef277", "id": "59b93afb-23d2-11e8-a6a3-ec086b02610b"},...]
"annotations": [{"image_id": "59ffbd00-23d2-11e8-a6a3-ec086b02610b", "category_id": 1, "bbox": [1118.72, 570.88, 328.96000000000004, 180.48000000000002], "id": "36132"}
I am trying to use the create_coco_tf_record.py file and adapt it. I do not have the 'iscrowd' or 'segmentations' in my annotations and a lot of the images do not have bounding boxes. I was wondering if someone has done similar and would be able to help please. Thanks! Here is the file...
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
r"""Convert raw COCO dataset to TFRecord for object_detection.
This tool supports data generation for object detection (boxes, masks),
keypoint detection, and DensePose.
Please note that this tool creates sharded output files.
Example usage:
python create_coco_tf_record.py --logtostderr \
--train_image_dir="${TRAIN_IMAGE_DIR}" \
--val_image_dir="${VAL_IMAGE_DIR}" \
--test_image_dir="${TEST_IMAGE_DIR}" \
--train_annotations_file="${TRAIN_ANNOTATIONS_FILE}" \
--val_annotations_file="${VAL_ANNOTATIONS_FILE}" \
--testdev_annotations_file="${TESTDEV_ANNOTATIONS_FILE}" \
--output_dir="${OUTPUT_DIR}"
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import hashlib
import io
import json
import logging
import os
import contextlib2
import numpy as np
import PIL.Image
from pycocotools import mask
import tensorflow.compat.v1 as tf
from object_detection.dataset_tools import tf_record_creation_util
from object_detection.utils import dataset_util
from object_detection.utils import label_map_util
flags = tf.app.flags
tf.flags.DEFINE_boolean(
'include_masks', False, 'Whether to include instance segmentations masks '
'(PNG encoded) in the result. default: False.')
tf.flags.DEFINE_string('train_image_dir', '', 'Training image directory.')
tf.flags.DEFINE_string('val_image_dir', '', 'Validation image directory.')
tf.flags.DEFINE_string('test_image_dir', '', 'Test image directory.')
tf.flags.DEFINE_string('train_annotations_file', '',
'Training annotations JSON file.')
tf.flags.DEFINE_string('val_annotations_file', '',
'Validation annotations JSON file.')
tf.flags.DEFINE_string('testdev_annotations_file', '',
'Test-dev annotations JSON file.')
tf.flags.DEFINE_string('train_keypoint_annotations_file', '',
'Training annotations JSON file.')
tf.flags.DEFINE_string('val_keypoint_annotations_file', '',
'Validation annotations JSON file.')
# DensePose is only available for coco 2014.
tf.flags.DEFINE_string('train_densepose_annotations_file', '',
'Training annotations JSON file for DensePose.')
tf.flags.DEFINE_string('val_densepose_annotations_file', '',
'Validation annotations JSON file for DensePose.')
tf.flags.DEFINE_string('output_dir', '/tmp/', 'Output data directory.')
# Whether to only produce images/annotations on person class (for keypoint /
# densepose task).
tf.flags.DEFINE_boolean('remove_non_person_annotations', False, 'Whether to '
'remove all annotations for non-person objects.')
tf.flags.DEFINE_boolean('remove_non_person_images', False, 'Whether to '
'remove all examples that do not contain a person.')
FLAGS = flags.FLAGS
logger = tf.get_logger()
logger.setLevel(logging.INFO)
_COCO_KEYPOINT_NAMES = [
b'nose', b'left_eye', b'right_eye', b'left_ear', b'right_ear',
b'left_shoulder', b'right_shoulder', b'left_elbow', b'right_elbow',
b'left_wrist', b'right_wrist', b'left_hip', b'right_hip',
b'left_knee', b'right_knee', b'left_ankle', b'right_ankle'
]
_COCO_PART_NAMES = [
b'torso_back', b'torso_front', b'right_hand', b'left_hand', b'left_foot',
b'right_foot', b'right_upper_leg_back', b'left_upper_leg_back',
b'right_upper_leg_front', b'left_upper_leg_front', b'right_lower_leg_back',
b'left_lower_leg_back', b'right_lower_leg_front', b'left_lower_leg_front',
b'left_upper_arm_back', b'right_upper_arm_back', b'left_upper_arm_front',
b'right_upper_arm_front', b'left_lower_arm_back', b'right_lower_arm_back',
b'left_lower_arm_front', b'right_lower_arm_front', b'right_face',
b'left_face',
]
_DP_PART_ID_OFFSET = 1
def clip_to_unit(x):
return min(max(x, 0.0), 1.0)
def create_tf_example(image,
annotations_list,
image_dir,
category_index,
include_masks=False,
keypoint_annotations_dict=None,
densepose_annotations_dict=None,
remove_non_person_annotations=False,
remove_non_person_images=False):
"""Converts image and annotations to a tf.Example proto.
Args:
image: dict with keys: [u'license', u'file_name', u'coco_url', u'height',
u'width', u'date_captured', u'flickr_url', u'id']
annotations_list:
list of dicts with keys: [u'segmentation', u'area', u'iscrowd',
u'image_id', u'bbox', u'category_id', u'id'] Notice that bounding box
coordinates in the official COCO dataset are given as [x, y, width,
height] tuples using absolute coordinates where x, y represent the
top-left (0-indexed) corner. This function converts to the format
expected by the Tensorflow Object Detection API (which is which is
[ymin, xmin, ymax, xmax] with coordinates normalized relative to image
size).
image_dir: directory containing the image files.
category_index: a dict containing COCO category information keyed by the
'id' field of each category. See the label_map_util.create_category_index
function.
include_masks: Whether to include instance segmentations masks
(PNG encoded) in the result. default: False.
keypoint_annotations_dict: A dictionary that maps from annotation_id to a
dictionary with keys: [u'keypoints', u'num_keypoints'] represeting the
keypoint information for this person object annotation. If None, then
no keypoint annotations will be populated.
densepose_annotations_dict: A dictionary that maps from annotation_id to a
dictionary with keys: [u'dp_I', u'dp_x', u'dp_y', 'dp_U', 'dp_V']
representing part surface coordinates. For more information see
http://densepose.org/.
remove_non_person_annotations: Whether to remove any annotations that are
not the "person" class.
remove_non_person_images: Whether to remove any images that do not contain
at least one "person" annotation.
Returns:
key: SHA256 hash of the image.
example: The converted tf.Example
num_annotations_skipped: Number of (invalid) annotations that were ignored.
num_keypoint_annotation_skipped: Number of keypoint annotations that were
skipped.
num_densepose_annotation_skipped: Number of DensePose annotations that were
skipped.
Raises:
ValueError: if the image pointed to by data['filename'] is not a valid JPEG
"""
image_height = image['height']
image_width = image['width']
filename = image['file_name']
image_id = image['id']
full_path = os.path.join(image_dir, filename)
with tf.gfile.GFile(full_path, 'rb') as fid:
encoded_jpg = fid.read()
encoded_jpg_io = io.BytesIO(encoded_jpg)
image = PIL.Image.open(encoded_jpg_io)
key = hashlib.sha256(encoded_jpg).hexdigest()
xmin = []
xmax = []
ymin = []
ymax = []
is_crowd = []
category_names = []
category_ids = []
area = []
encoded_mask_png = []
keypoints_x = []
keypoints_y = []
keypoints_visibility = []
keypoints_name = []
num_keypoints = []
include_keypoint = keypoint_annotations_dict is not None
num_annotations_skipped = 0
num_keypoint_annotation_used = 0
num_keypoint_annotation_skipped = 0
dp_part_index = []
dp_x = []
dp_y = []
dp_u = []
dp_v = []
dp_num_points = []
densepose_keys = ['dp_I', 'dp_U', 'dp_V', 'dp_x', 'dp_y', 'bbox']
include_densepose = densepose_annotations_dict is not None
num_densepose_annotation_used = 0
num_densepose_annotation_skipped = 0
for object_annotations in annotations_list:
(x, y, width, height) = tuple(object_annotations['bbox'])
if width <= 0 or height <= 0:
num_annotations_skipped += 1
continue
if x + width > image_width or y + height > image_height:
num_annotations_skipped += 1
continue
category_id = int(object_annotations['category_id'])
category_name = category_index[category_id]['name'].encode('utf8')
if remove_non_person_annotations and category_name != b'person':
num_annotations_skipped += 1
continue
xmin.append(float(x) / image_width)
xmax.append(float(x + width) / image_width)
ymin.append(float(y) / image_height)
ymax.append(float(y + height) / image_height)
#is_crowd.append(object_annotations['iscrowd'])
category_ids.append(category_id)
category_names.append(category_name)
#area.append(object_annotations['area'])
# if include_masks:
# run_len_encoding = mask.frPyObjects(object_annotations['segmentation'],
# image_height, image_width)
# binary_mask = mask.decode(run_len_encoding)
# if not object_annotations['iscrowd']:
# binary_mask = np.amax(binary_mask, axis=2)
# pil_image = PIL.Image.fromarray(binary_mask)
# output_io = io.BytesIO()
# pil_image.save(output_io, format='PNG')
# encoded_mask_png.append(output_io.getvalue())
# if include_keypoint:
# annotation_id = object_annotations['id']
# if annotation_id in keypoint_annotations_dict:
# num_keypoint_annotation_used += 1
# keypoint_annotations = keypoint_annotations_dict[annotation_id]
# keypoints = keypoint_annotations['keypoints']
# num_kpts = keypoint_annotations['num_keypoints']
# keypoints_x_abs = keypoints[::3]
# keypoints_x.extend(
# [float(x_abs) / image_width for x_abs in keypoints_x_abs])
# keypoints_y_abs = keypoints[1::3]
# keypoints_y.extend(
# [float(y_abs) / image_height for y_abs in keypoints_y_abs])
# keypoints_visibility.extend(keypoints[2::3])
# keypoints_name.extend(_COCO_KEYPOINT_NAMES)
# num_keypoints.append(num_kpts)
# else:
# keypoints_x.extend([0.0] * len(_COCO_KEYPOINT_NAMES))
# keypoints_y.extend([0.0] * len(_COCO_KEYPOINT_NAMES))
# keypoints_visibility.extend([0] * len(_COCO_KEYPOINT_NAMES))
# keypoints_name.extend(_COCO_KEYPOINT_NAMES)
# num_keypoints.append(0)
# if include_densepose:
# annotation_id = object_annotations['id']
# if (annotation_id in densepose_annotations_dict and
# all(key in densepose_annotations_dict[annotation_id]
# for key in densepose_keys)):
# dp_annotations = densepose_annotations_dict[annotation_id]
# num_densepose_annotation_used += 1
# dp_num_points.append(len(dp_annotations['dp_I']))
# dp_part_index.extend([int(i - _DP_PART_ID_OFFSET)
# for i in dp_annotations['dp_I']])
# # DensePose surface coordinates are defined on a [256, 256] grid
# # relative to each instance box (i.e. absolute coordinates in range
# # [0., 256.]). The following converts the coordinates
# # so that they are expressed in normalized image coordinates.
# dp_x_box_rel = [
# clip_to_unit(val / 256.) for val in dp_annotations['dp_x']]
# dp_x_norm = [(float(x) + x_box_rel * width) / image_width
# for x_box_rel in dp_x_box_rel]
# dp_y_box_rel = [
# clip_to_unit(val / 256.) for val in dp_annotations['dp_y']]
# dp_y_norm = [(float(y) + y_box_rel * height) / image_height
# for y_box_rel in dp_y_box_rel]
# dp_x.extend(dp_x_norm)
# dp_y.extend(dp_y_norm)
# dp_u.extend(dp_annotations['dp_U'])
# dp_v.extend(dp_annotations['dp_V'])
# else:
# dp_num_points.append(0)
# if (remove_non_person_images and
# not any(name == b'person' for name in category_names)):
# return (key, None, num_annotations_skipped,
# num_keypoint_annotation_skipped, num_densepose_annotation_skipped)
feature_dict = {
'image/height':
dataset_util.int64_feature(image_height),
'image/width':
dataset_util.int64_feature(image_width),
'image/filename':
dataset_util.bytes_feature(filename.encode('utf8')),
'image/source_id':
dataset_util.bytes_feature(str(image_id).encode('utf8')),
'image/key/sha256':
dataset_util.bytes_feature(key.encode('utf8')),
'image/encoded':
dataset_util.bytes_feature(encoded_jpg),
'image/format':
dataset_util.bytes_feature('jpeg'.encode('utf8')),
'image/object/bbox/xmin':
dataset_util.float_list_feature(xmin),
'image/object/bbox/xmax':
dataset_util.float_list_feature(xmax),
'image/object/bbox/ymin':
dataset_util.float_list_feature(ymin),
'image/object/bbox/ymax':
dataset_util.float_list_feature(ymax),
'image/object/class/text':
dataset_util.bytes_list_feature(category_names),
'image/object/is_crowd':
dataset_util.int64_list_feature(is_crowd),
'image/object/area':
dataset_util.float_list_feature(area),
}
# if include_masks:
# feature_dict['image/object/mask'] = (
# dataset_util.bytes_list_feature(encoded_mask_png))
# if include_keypoint:
# feature_dict['image/object/keypoint/x'] = (
# dataset_util.float_list_feature(keypoints_x))
# feature_dict['image/object/keypoint/y'] = (
# dataset_util.float_list_feature(keypoints_y))
# feature_dict['image/object/keypoint/num'] = (
# dataset_util.int64_list_feature(num_keypoints))
# feature_dict['image/object/keypoint/visibility'] = (
# dataset_util.int64_list_feature(keypoints_visibility))
# feature_dict['image/object/keypoint/text'] = (
# dataset_util.bytes_list_feature(keypoints_name))
# num_keypoint_annotation_skipped = (
# len(keypoint_annotations_dict) - num_keypoint_annotation_used)
# if include_densepose:
# feature_dict['image/object/densepose/num'] = (
# dataset_util.int64_list_feature(dp_num_points))
# feature_dict['image/object/densepose/part_index'] = (
# dataset_util.int64_list_feature(dp_part_index))
# feature_dict['image/object/densepose/x'] = (
# dataset_util.float_list_feature(dp_x))
# feature_dict['image/object/densepose/y'] = (
# dataset_util.float_list_feature(dp_y))
# feature_dict['image/object/densepose/u'] = (
# dataset_util.float_list_feature(dp_u))
# feature_dict['image/object/densepose/v'] = (
# dataset_util.float_list_feature(dp_v))
# num_densepose_annotation_skipped = (
# len(densepose_annotations_dict) - num_densepose_annotation_used)
example = tf.train.Example(features=tf.train.Features(feature=feature_dict))
return (key, example, num_annotations_skipped,
num_keypoint_annotation_skipped, num_densepose_annotation_skipped)
def _create_tf_record_from_coco_annotations(annotations_file, image_dir,
output_path, include_masks,
num_shards,
keypoint_annotations_file='',
densepose_annotations_file='',
remove_non_person_annotations=False,
remove_non_person_images=False):
"""Loads COCO annotation json files and converts to tf.Record format.
Args:
annotations_file: JSON file containing bounding box annotations.
image_dir: Directory containing the image files.
output_path: Path to output tf.Record file.
include_masks: Whether to include instance segmentations masks
(PNG encoded) in the result. default: False.
num_shards: number of output file shards.
keypoint_annotations_file: JSON file containing the person keypoint
annotations. If empty, then no person keypoint annotations will be
generated.
densepose_annotations_file: JSON file containing the DensePose annotations.
If empty, then no DensePose annotations will be generated.
remove_non_person_annotations: Whether to remove any annotations that are
not the "person" class.
remove_non_person_images: Whether to remove any images that do not contain
at least one "person" annotation.
"""
with contextlib2.ExitStack() as tf_record_close_stack, \
tf.gfile.GFile(annotations_file, 'r') as fid:
output_tfrecords = tf_record_creation_util.open_sharded_output_tfrecords(
tf_record_close_stack, output_path, num_shards)
groundtruth_data = json.load(fid)
images = groundtruth_data['images']
category_index = label_map_util.create_category_index(
groundtruth_data['categories'])
annotations_index = {}
if 'annotations' in groundtruth_data:
logging.info('Found groundtruth annotations. Building annotations index.')
for annotation in groundtruth_data['annotations']:
image_id = annotation['image_id']
if image_id not in annotations_index:
annotations_index[image_id] = []
annotations_index[image_id].append(annotation)
missing_annotation_count = 0
for image in images:
image_id = image['id']
if image_id not in annotations_index:
missing_annotation_count += 1
annotations_index[image_id] = []
logging.info('%d images are missing annotations.',
missing_annotation_count)
keypoint_annotations_index = {}
if keypoint_annotations_file:
with tf.gfile.GFile(keypoint_annotations_file, 'r') as kid:
keypoint_groundtruth_data = json.load(kid)
if 'annotations' in keypoint_groundtruth_data:
for annotation in keypoint_groundtruth_data['annotations']:
image_id = annotation['image_id']
if image_id not in keypoint_annotations_index:
keypoint_annotations_index[image_id] = {}
keypoint_annotations_index[image_id][annotation['id']] = annotation
densepose_annotations_index = {}
if densepose_annotations_file:
with tf.gfile.GFile(densepose_annotations_file, 'r') as fid:
densepose_groundtruth_data = json.load(fid)
if 'annotations' in densepose_groundtruth_data:
for annotation in densepose_groundtruth_data['annotations']:
image_id = annotation['image_id']
if image_id not in densepose_annotations_index:
densepose_annotations_index[image_id] = {}
densepose_annotations_index[image_id][annotation['id']] = annotation
total_num_annotations_skipped = 0
total_num_keypoint_annotations_skipped = 0
total_num_densepose_annotations_skipped = 0
for idx, image in enumerate(images):
if idx % 100 == 0:
logging.info('On image %d of %d', idx, len(images))
annotations_list = annotations_index[image['id']]
keypoint_annotations_dict = None
if keypoint_annotations_file:
keypoint_annotations_dict = {}
if image['id'] in keypoint_annotations_index:
keypoint_annotations_dict = keypoint_annotations_index[image['id']]
densepose_annotations_dict = None
if densepose_annotations_file:
densepose_annotations_dict = {}
if image['id'] in densepose_annotations_index:
densepose_annotations_dict = densepose_annotations_index[image['id']]
(_, tf_example, num_annotations_skipped, num_keypoint_annotations_skipped,
num_densepose_annotations_skipped) = create_tf_example(
image, annotations_list, image_dir, category_index, include_masks,
keypoint_annotations_dict, densepose_annotations_dict,
remove_non_person_annotations, remove_non_person_images)
total_num_annotations_skipped += num_annotations_skipped
total_num_keypoint_annotations_skipped += num_keypoint_annotations_skipped
total_num_densepose_annotations_skipped += (
num_densepose_annotations_skipped)
shard_idx = idx % num_shards
if tf_example:
output_tfrecords[shard_idx].write(tf_example.SerializeToString())
logging.info('Finished writing, skipped %d annotations.',
total_num_annotations_skipped)
if keypoint_annotations_file:
logging.info('Finished writing, skipped %d keypoint annotations.',
total_num_keypoint_annotations_skipped)
if densepose_annotations_file:
logging.info('Finished writing, skipped %d DensePose annotations.',
total_num_densepose_annotations_skipped)
def main(_):
assert FLAGS.train_image_dir, '`train_image_dir` missing.'
assert FLAGS.val_image_dir, '`val_image_dir` missing.'
assert FLAGS.test_image_dir, '`test_image_dir` missing.'
assert FLAGS.train_annotations_file, '`train_annotations_file` missing.'
assert FLAGS.val_annotations_file, '`val_annotations_file` missing.'
assert FLAGS.testdev_annotations_file, '`testdev_annotations_file` missing.'
if not tf.gfile.IsDirectory(FLAGS.output_dir):
tf.gfile.MakeDirs(FLAGS.output_dir)
train_output_path = os.path.join(FLAGS.output_dir, 'coco_train.record')
val_output_path = os.path.join(FLAGS.output_dir, 'coco_val.record')
testdev_output_path = os.path.join(FLAGS.output_dir, 'coco_testdev.record')
_create_tf_record_from_coco_annotations(
FLAGS.train_annotations_file,
FLAGS.train_image_dir,
train_output_path,
FLAGS.include_masks,
num_shards=100,
keypoint_annotations_file=FLAGS.train_keypoint_annotations_file,
densepose_annotations_file=FLAGS.train_densepose_annotations_file,
remove_non_person_annotations=FLAGS.remove_non_person_annotations,
remove_non_person_images=FLAGS.remove_non_person_images)
_create_tf_record_from_coco_annotations(
FLAGS.val_annotations_file,
FLAGS.val_image_dir,
val_output_path,
FLAGS.include_masks,
num_shards=50,
keypoint_annotations_file=FLAGS.val_keypoint_annotations_file,
densepose_annotations_file=FLAGS.val_densepose_annotations_file,
remove_non_person_annotations=FLAGS.remove_non_person_annotations,
remove_non_person_images=FLAGS.remove_non_person_images)
_create_tf_record_from_coco_annotations(
FLAGS.testdev_annotations_file,
FLAGS.test_image_dir,
testdev_output_path,
FLAGS.include_masks,
num_shards=50)
if __name__ == '__main__':
tf.app.run()
You can leave the iscrowd and segmentations as empty if you are doing object detection or classification or ... tasks. But You wouldn't make use of the data with missing bounding boxes for such tasks.
I have two csv files that i am uploading from an ec2 instance to the s3 bucket along with a few other files. All the other files are being uploaded just fine but my csv files, though it is uploaded, there seems ot be no data inside it even though the local copy of the file on the instance is showing the data. im not sure why its saying 0 bytes on the bucket.
the csv file is part of another larger program. here is the code.
from boto3.session import Session
import botocore
import boto3
import zipfile
import darknet
import os
import cv2
import glob
import csv
import numpy as np
global lat_start, lon_start
import shutil
#HELPER FUNCTION DEFINITIONS
ACCESS_KEY = '*********'
SECRET_KEY = '******D'
def image_detection(image_path, network, class_names, class_colors, thresh):
# Darknet doesn't accept numpy images.
# Create one with image we reuse for each detect
width = darknet.network_width(network)
height = darknet.network_height(network)
darknet_image = darknet.make_image(width, height, 3)
image = cv2.imread(image_path)
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image_resized = cv2.resize(image_rgb, (width, height),interpolation=cv2.INTER_LINEAR)
darknet.copy_image_from_bytes(darknet_image, image_resized.tobytes())
detections = darknet.detect_image(network, class_names, darknet_image, thresh=thresh)
darknet.free_image(darknet_image)
image = darknet.draw_boxes(detections, image_resized, class_colors)
return cv2.cvtColor(image, cv2.COLOR_BGR2RGB), detections
def discretize_line(lat_start, lon_start, d_element, d, bearing):
# d_element -> how many element we need in a line secment
# global lat_start, lon_start
R = 6371.0*1000.0
# -1 because in case of 10 elements/points we also want len(lat_array) the same
dstep = d/(d_element-1) #0.6524896365354135 #2.0 # meters
dist_list = np.ones(int(d/dstep))*dstep
# print(dist_list)
brg = np.radians(bearing)
# if d%dstep != 0:
# dist_list = np.append(dist_list, d%dstep)
# This will append lat and lon into array which contains
# small segments of distance
lat_array = np.array([np.radians(lat_start)]) # rads
lon_array = np.array([np.radians(lon_start)]) # rads
# lat_array = np.array([])
# lon_array = np.array([])
for i, dist in enumerate(dist_list):
## last element make the waypoint shifted, so we break it
if i >= (d_element):
break
lat1 = lat_array[i]
lon1 = lon_array[i]
# print(dist)
Ad = dist/R
lat2 = np.arcsin(np.sin(lat1)*np.cos(Ad) + np.cos(lat1)*np.sin(Ad)*np.cos(brg))
lon2 = lon1 + np.arctan2( (np.sin(brg)*np.sin(Ad)*np.cos(lat1)) , (np.cos(Ad) - np.sin(lat1)*np.sin(lat2)))
lat_array = np.append(lat_array, lat2)
lon_array = np.append(lon_array, lon2)
# print(i)
return lat_array, lon_array
def get_distance_bearing(lat1, lon1, lat2, lon2):
# global lat_start, lon_start
R = 6371.0*1000.0
lat_start = np.radians(lat1)
lon_start = np.radians(lon1)
lat_end = np.radians(lat2)
lon_end = np.radians(lon2)
dLat = lat_end - lat_start
dLon = lon_end - lon_start
a = np.sin(dLat/2.0)*np.sin(dLat/2.0) + np.cos(lat_start)*np.cos(lat_end)*np.sin(dLon/2.0)*np.sin(dLon/2.0)
c = 2.0*np.arctan2(np.sqrt(a),np.sqrt(1-a))
d = c*R
y = np.sin(dLon)*np.cos(lat_end)
x = np.cos(lat_start)*np.sin(lat_end) - np.sin(lat_start)*np.cos(lat_end)*np.cos(dLon)
bearing = np.degrees(np.arctan2(y,x))
return d, bearing
def upload_to_aws(local_file, bucket, s3_file):
s3 = boto3.client('s3', aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY)
try:
s3.upload_file(local_file, bucket, s3_file)
print("Upload Successful")
return True
except FileNotFoundError:
print("The file was not found")
return False
except NoCredentialsError:
print("Credentials not available")
return False
##END OF FUNCTION DEFINITIONS ##
#Unzip the zip file and its contents
print("unzipping")
path_to_zip_file = "/home/ubuntu/pano/Zip/Videos.zip"
with zipfile.ZipFile(path_to_zip_file, 'r') as zip_ref:
zip_ref.extractall("/home/ubuntu/pano/Video")
print("Finished Unzipping")
#End of Unzip
# CSV open and declaration##
data_file_path = "/home/ubuntu/pano/stack/quantity.csv"
data_file = open(data_file_path, "w+")
dataCSVWriter = csv.writer(data_file, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
dataCSVWriter.writerow(['lat', 'lon', 'Quantity'])
#CSV for lane thumbnail
thumbnail_data_file_path = "/home/ubuntu/pano/stack/lane_thumbnail.csv"
thumbnail_data_file = open(thumbnail_data_file_path, "w+")
thumbnail_dataCSVWriter = csv.writer(thumbnail_data_file, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
thumbnail_dataCSVWriter.writerow(['lat', 'lon'])
#Define start and end point lists
#start_point_list = [(35.841454251754755, 139.52427014959153),(35.84147944801779, 139.52420150963678)]
start_point_list = [(36.12083710338884, 139.21630320454503),(36.12080527337101, 139.2164926108044)]
#end_point_list = [(35.84151350159559, 139.52424466860762),(35.84144222040454, 139.52422739581436)]
end_point_list = [(36.12083735438514, 139.2164757318577),(36.12081575161991, 139.21630345327617)]
wp_lat_array = np.array([])
wp_lon_array = np.array([])
##Split th eline into points and it is stored in lat array lon array
"""for i in range(len(start_point_list)):
## input two points and find a slicing waypoint between it
distance, bearing_deg = get_distance_bearing(start_point_list[i][0], start_point_list[i][1], end_point_list[i][0], end_point_list[i][1])
print(distance)
lat_array, lon_array = discretize_line(start_point_list[i][0], start_point_list[i][1], float(d_element[i]), distance, bearing_deg)"""
#Initialize the detector variables and paths
quantity_bottles_frame = []
config_file = "/home/ubuntu/darknet_bottle_example/yolov4_bottle_can.cfg"
data_file = "/home/ubuntu/darknet_bottle_example/obj_bottle_can.data"
weights = "/home/ubuntu/darknet_bottle_example/yolov4_bottle_can_best.weights"
network, class_names, class_colors = darknet.load_network(
config_file,
data_file,
weights,
batch_size=1
)
image_dir = "/home/ubuntu/pano/Frames"
#1.Split into frames
path = "/home/ubuntu/pano/Video/Panorama/Videos"
j = 0
"""Order of events
1. Split into frames
2. Rotate images if needed
3. Running through detctor
4. Calculate count and draw bounding boxes
5. Store these images in respective directoies
6. Take start point of lane and end point and split into many coordinates in between based on number of frames
7. Write to csv file
8. Stack the images per lane
9. Empty the Frames folder after every lane
10. Upload stacked images and csv to cloud """
# Parameter to change is fps in the ffmpeg command. Change accoprding to need based on reference
for filename in os.listdir(path):
if (filename.endswith(".mp4")): #or .avi, .mpeg, whatever.
j += 1
path1 = path + filename
print(path1)
os.system("ffmpeg -i /home/ubuntu/pano/Video/Panorama/Videos/{0} -vf fps=0.07 /home/ubuntu/pano/Frames/{1}-%3d.jpg".format(filename,j))
#2. Rotate images if needed
frames_path = "/home/ubuntu/pano/Frames/*.jpg"
list_images = glob.glob(frames_path)
list_sorted = sorted(list_images)
#for image in list_sorted:
#read the image
# temp = cv2.imread(image)
# image1 = cv2.rotate(temp, cv2.ROTATE_90_COUNTERCLOCKWISE)
# cv2.imwrite("{0}".format(image), image1)
## according to how many partial panorama we have in each lane
d_element =[len(list_images)]
print(f"Now detecting objects in lane {j}")
#3. Running through detctor
frame_number = 1
for image in sorted(os.listdir(image_dir)):
#Path to the input images for the detector i.e Frames
quantity_frame = 0
image_name = f"{image}"
ext = '.jpg'
input_image_name = image_name
image_path = os.path.join(image_dir, input_image_name)
print(image_path)
#Path to output images to be stored after running through detector
output_dir = f"/home/ubuntu/pano/lane{j}"
output_name = "yolo_" + image_name
output_path = os.path.join(output_dir, output_name)
# image = load_images(image_path)
dn_frame_width = 416
dn_frame_height = 416
frame = cv2.imread(image_path)
frame_width = frame.shape[1]
frame_height = frame.shape[0]
#### Passing the image to darknet
image, detections = image_detection(image_path, network, class_names, class_colors, thresh=0.05)
#cv2.imwrite(f'/home/ubuntu/temp/Inference{frame_number}.jpg', image)
#cv2.imwrite(f'/home/ubuntu/temp/orignal_detect{frame_number}.jpg', frame)
###Based on the detections, running them through a loop to draw bounding box and also incrememnt count of object in the frame
#4. Calculate count and draw bounding boxes
for i in range(len(detections)):
xc_percent = detections[i][2][0]/dn_frame_width
yc_percent = detections[i][2][1]/dn_frame_height
w_percent = detections[i][2][2]/dn_frame_width
h_percent = detections[i][2][3]/dn_frame_height
xc = xc_percent*frame_width
yc = yc_percent*frame_height
w = w_percent*frame_width
h = h_percent*frame_height
xmin = xc - w/2.0
ymin = yc - h/2.0
xmax = xc + w/2.0
ymax = yc + h/2.0
#If object is detected, increase the count of the object in the frame
if detections[i][0] == "bottle":
cv2.rectangle(frame, (int(xmin),int(ymin)),(int(xmax),int(ymax)),(0,0,255),2)
cv2.putText(frame, "bottle", (int(xmin), int(ymin-10)), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0,0,255), 2)
quantity_frame += 1
elif detections[i][0] == "can":
cv2.rectangle(frame, (int(xmin),int(ymin)),(int(xmax),int(ymax)),(255,0,0),2)
cv2.putText(frame, "can", (int(xmin), int(ymin-10)), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255,0,0), 2)
else:
print(f"{image} has no objects ")
print(f"Quantity in frame {frame_number} = {quantity_frame}")
#5. Store these images in respective directoies
cv2.imwrite(output_path, frame)
quantity_bottles_frame.append(quantity_frame)
frame_number += 1
###Split the points into equidistant points between start point and end point
##6. Take start point of lane and end point and split into many coordinates in between based on number of frames
distance, bearing_deg = get_distance_bearing(start_point_list[j-1][0], start_point_list[j-1][1], end_point_list[j-1][0], end_point_list[j-1][1])
print(distance)
lat_array, lon_array = discretize_line(start_point_list[j-1][0], start_point_list[j-1][1], float(d_element[0]), distance, bearing_deg)
lat_csv = []
lon_csv = []
##Convery those points into degrees
for lat,lon in zip(lat_array, lon_array):
lat_degrees = "{:}".format(np.degrees(lat))
lon_degrees = "{:}".format(np.degrees(lon))
lat_csv.append(lat_degrees)
lon_csv.append(lon_degrees)
#lat_csv = "{:}".format(np.degrees(lat))
#lon_csv = "{:}".format(np.degrees(lon))
##7.Write each row in the csv file
for k in range(d_element[0]):
dataCSVWriter.writerow([lat_csv[k], lon_csv[k], quantity_bottles_frame[k]])
#if k != d_element[0]-1:
# dataCSVWriter.writerow([lat_csv[k], lon_csv[k], quantity_bottles_frame[k], "-", "-" ])
if k ==d_element[0]-1:
print(lat_csv[int(d_element[0]/2)])
thumbnail_dataCSVWriter.writerow([ lat_csv[int(d_element[0]/2)],lon_csv[int(d_element[0]/2)]])
#####8.STACKING THE IMAGES ######
images = []
stacking_input = f"/home/ubuntu/pano/lane{j}/*.jpg"
list_images = glob.glob(stacking_input)
#print(list_images)
stacking_input_reverse = sorted(list_images, reverse = True)
print(stacking_input_reverse)
for image in stacking_input_reverse:
img = cv2.imread(image)
images.append(img)
final_image = cv2.hconcat(images)
image_name = f"cloud_lane{j}_stack.jpg"
stacking_output = f"/home/ubuntu/pano/stack"
output_path = os.path.join(stacking_output, image_name)
cv2.imwrite(output_path, final_image)
##### 9. DELETE FRAMES AFTER ONE ITERATION OF LOOP #####
for f in os.listdir(image_dir):
del_path = "/home/ubuntu/pano/Frames/" + f
os.remove(del_path)
else:
continue
#Close csv file
#data_file.close()
#thumbnail_data_file.close()
### 10. Upload to s3 bucket ####
stack_path = "/home/ubuntu/pano/stack"
for file in sorted(os.listdir(stack_path)):
print(f"Uploading {file}")
uploaded = upload_to_aws(f'/home/ubuntu/pano/stack/{file}', 'fbt-pano-test', f'{file}')
Do i need to close the csv file in any way? Or does s3 not support csv upload through boto3?
I found it. Turns out, the csv files werent closed at the end. So i moved the upload to s3 part to another program. now python closes the csv files at the end of this program automatically. and so when the upload program runs next, it gets uploaded properly.
I am trying to make contour plot from unstructured grid. My code does not work and I could not find an example to follow. I did not include the input file, hoping that a sample unstructured grid is easy to find. I managed to make work on Paraview by converting cell data to point data and then contouring "p" scalar. But I cannot do it with vtk. What's wrong in my code?
from vtk import *
file_name = "results.vtk"
reader = vtkUnstructuredGridReader()
reader.SetFileName(file_name)
reader.Update()
output = reader.GetOutput()
scalar_range = output.GetScalarRange()
c2p = vtkCellDataToPointData()
c2p.SetInputData(output)
contours = vtkContourGrid()
contours.SetInputData(c2p.GetOutput())
contours.SetValue(0, 0.007009505294263363)
gridmapper = vtkDataSetMapper()
gridmapper.SetInputData(output)
gridmapper.GetInput().GetCellData().SetActiveScalars("p")
gridmapper.SetScalarVisibility(1)
gridmapper.SetScalarRange(scalar_range)
mapper = vtkPolyDataMapper()
#mapper = vtkDataSetMapper()
mapper.SetInputConnection(contours.GetOutputPort())
actor = vtkActor()
actor.SetMapper(mapper)
gridactor = vtkActor()
gridactor.SetMapper(gridmapper)
gridactor.GetProperty().EdgeVisibilityOn()
renderer = vtkRenderer()
renderer.AddActor(actor)
renderer.AddActor(gridactor)
#renderer.SetBackground(1, 1, 1) # Set background to white
renderer_window = vtkRenderWindow()
renderer_window.AddRenderer(renderer)
interactor = vtkRenderWindowInteractor()
interactor.SetRenderWindow(renderer_window)
interactor.Initialize()
interactor.Start()
I'm using a script I found online to convert some files through parsing some XML. The script was built in Python 2.6 and it's using a module that I believe doesn't come with 2.6 through what I've read on the web. I'm wondering if there's a work around. The error I am getting is:
No Module name EXT
In the following script, I think it's getting hung up on import xml.dom.ext and it only seems to use this object at the very end to PrettyPrint (See the very last Try statement) I'm wondering if there's a workaround for this in 2.6? I can't seem to find a module that contains the EXT object which I can import.
The script is:
from xml.dom.minidom import Document
import xml.dom.ext
import string
import os
import arcpy
#Read input parameters from GP dialog
output = arcpy.GetParameterAsText(0)
#Create an output qgs file
f = open(output, "w")
# Create the minidom
doc = Document()
# Create the <qgis> base element
qgis = doc.createElement("qgis")
qgis.setAttribute("projectname", " ")
qgis.setAttribute("version", "1.6.0-Capiapo")
doc.appendChild(qgis)
# Create the <title> element
title = doc.createElement("title")
qgis.appendChild(title)
# Assign current document
mxd = arcpy.mapping.MapDocument("CURRENT")
print 'Converting mxd........'
# Dataframe elements
df = arcpy.mapping.ListDataFrames(mxd)[0]
unit = doc.createTextNode(df.mapUnits)
xmin1 = doc.createTextNode(str(df.extent.XMin))
ymin1 = doc.createTextNode(str(df.extent.YMin))
xmax1 = doc.createTextNode(str(df.extent.XMax))
ymax1 = doc.createTextNode(str(df.extent.YMax))
# srsid = doc.createTextNode
srid1 = doc.createTextNode(str(df.spatialReference.factoryCode))
srid2 = doc.createTextNode(str(df.spatialReference.factoryCode))
epsg1 = doc.createTextNode(str(df.spatialReference.factoryCode))
epsg2 = doc.createTextNode(str(df.spatialReference.factoryCode))
description1 = doc.createTextNode(str(df.spatialReference.name))
description2 = doc.createTextNode(str(df.spatialReference.name))
ellipsoidacronym1 = doc.createTextNode(str(df.spatialReference.name))
ellipsoidacronym2 = doc.createTextNode(str(df.spatialReference.name))
geographicflag1 = doc.createTextNode("true")
geographicflag2 = doc.createTextNode("true")
authid2 = doc.createTextNode("EPSG:"+str(df.spatialReference.factoryCode))
authid3 = doc.createTextNode("EPSG:"+str(df.spatialReference.factoryCode))
# Layerlist elements
lyrlist = arcpy.mapping.ListLayers(df)
count1 = str(len(lyrlist))
# mapcanvas
def map_canvas():
# Create the <mapcanvas> element
mapcanvas = doc.createElement("mapcanvas")
qgis.appendChild(mapcanvas)
# Create the <units> element
units = doc.createElement("units")
units.appendChild(unit)
mapcanvas.appendChild(units)
# Create the <extent> element
extent = doc.createElement("extent")
mapcanvas.appendChild(extent)
# Create the <xmin> element
xmin = doc.createElement("xmin")
xmin.appendChild(xmin1)
extent.appendChild(xmin)
# Create the <ymin> element
ymin = doc.createElement("ymin")
ymin.appendChild(ymin1)
extent.appendChild(ymin)
# Create the <xmax> element
xmax = doc.createElement("xmax")
xmax.appendChild(xmax1)
extent.appendChild(xmax)
# Create the <ymax> element
ymax = doc.createElement("ymax")
ymax.appendChild(ymax1)
extent.appendChild(ymax)
# Create the <projections> element
projections = doc.createElement("projections")
mapcanvas.appendChild(projections)
# Create the <destinationsrs> element
destinationsrs = doc.createElement("destinationsrs")
mapcanvas.appendChild(destinationsrs)
# Create the <spatialrefsys> element
spatialrefsys = doc.createElement("spatialrefsys")
destinationsrs.appendChild(spatialrefsys)
# Create the <proj4> element
proj4 = doc.createElement("proj4")
spatialrefsys.appendChild(proj4)
# Create the <srsid> element
srsid = doc.createElement("srsid")
spatialrefsys.appendChild(srsid)
# Create the <srid> element
srid = doc.createElement("srid")
srid.appendChild(srid1)
spatialrefsys.appendChild(srid)
# Create the <authid> element
authid = doc.createElement("authid")
authid.appendChild(authid2)
spatialrefsys.appendChild(authid)
# Create the <description> element
description = doc.createElement("description")
description.appendChild(description1)
spatialrefsys.appendChild(description)
# Create the <projectionacronym> element
projectionacronym = doc.createElement("projectionacronym")
spatialrefsys.appendChild(projectionacronym)
# Create the <ellipsoidacronym element
ellipsoidacronym = doc.createElement("ellipsoidacronym")
ellipsoidacronym.appendChild(ellipsoidacronym1)
spatialrefsys.appendChild(ellipsoidacronym)
# Create the <geographicflag> element
geographicflag = doc.createElement("geographicflag")
geographicflag.appendChild(geographicflag1)
spatialrefsys.appendChild(geographicflag)
# Legend
def legend_func():
# Create the <legend> element
legend = doc.createElement("legend")
qgis.appendChild(legend)
for lyr in lyrlist:
if(lyr.isGroupLayer == False):
# Create the <legendlayer> element
legendlayer = doc.createElement("legendlayer")
legendlayer.setAttribute("open", "true")
legendlayer.setAttribute("checked", "Qt::Checked")
legendlayer.setAttribute("name",str(lyr.name))
legend.appendChild(legendlayer)
# Create the <filegroup> element
filegroup = doc.createElement("filegroup")
filegroup.setAttribute("open", "true")
filegroup.setAttribute("hidden", "false")
legendlayer.appendChild(filegroup)
# Create the <legendlayerfile> element
legendlayerfile = doc.createElement("legendlayerfile")
legendlayerfile.setAttribute("isInOverview", "0")
legendlayerfile.setAttribute("layerid", str(lyr.name)+str(20110427170816078))
legendlayerfile.setAttribute("visible", "1")
filegroup.appendChild(legendlayerfile)
# Project Layers
def project_layers():
# Create the <projectlayers> element
projectlayers = doc.createElement("projectlayers")
projectlayers.setAttribute("layercount", count1)
qgis.appendChild(projectlayers)
for lyr in lyrlist:
if(lyr.isGroupLayer == False and lyr.isRasterLayer == False):
geometry1 = arcpy.Describe(lyr)
geometry2 = str(geometry1.shapeType)
ds = doc.createTextNode(str(lyr.dataSource))
name1 = doc.createTextNode(str(lyr.name)+str(20110427170816078))
name2 = doc.createTextNode(str(lyr.name))
# Create the <maplayer> element
maplayer = doc.createElement("maplayer")
maplayer.setAttribute("minimumScale", "0")
maplayer.setAttribute("maximumScale", "1e+08")
maplayer.setAttribute("minLabelScale", "0")
maplayer.setAttribute("maxLabelScale", "1e+08")
maplayer.setAttribute("geometry", geometry2)
if(lyr.isRasterLayer == True):
maplayer.setAttribute("type", "raster")
else:
maplayer.setAttribute("type", "vector")
maplayer.setAttribute("hasScaleBasedVisibilityFlag", "0")
maplayer.setAttribute("scaleBasedLabelVisibilityFlag", "0")
projectlayers.appendChild(maplayer)
# Create the <id> element
id = doc.createElement("id")
id.appendChild(name1)
maplayer.appendChild(id)
# Create the <datasource> element
datasource = doc.createElement("datasource")
datasource.appendChild(ds)
maplayer.appendChild(datasource)
# Create the <layername> element
layername = doc.createElement("layername")
layername.appendChild(name2)
maplayer.appendChild(layername)
# Create the <srs> element
srs = doc.createElement("srs")
maplayer.appendChild(srs)
# Create the <spatialrefsys> element
spatialrefsys = doc.createElement("spatialrefsys")
srs.appendChild(spatialrefsys)
# Create the <proj4> element
proj4 = doc.createElement("proj4")
spatialrefsys.appendChild(proj4)
# Create the <srsid> element
srsid = doc.createElement("srsid")
spatialrefsys.appendChild(srsid)
# Create the <srid> element
srid = doc.createElement("srid")
srid.appendChild(srid2)
spatialrefsys.appendChild(srid)
# Create the <authid> element
authid = doc.createElement("authid")
authid.appendChild(authid3)
spatialrefsys.appendChild(authid)
# Create the <description> element
description = doc.createElement("description")
description.appendChild(description2)
spatialrefsys.appendChild(description)
# Create the <projectionacronym> element
projectionacronym = doc.createElement("projectionacronym")
spatialrefsys.appendChild(projectionacronym)
# Create the <ellipsoidacronym element
ellipsoidacronym = doc.createElement("ellipsoidacronym")
ellipsoidacronym.appendChild(ellipsoidacronym2)
spatialrefsys.appendChild(ellipsoidacronym)
# Create the <geographicflag> element
geographicflag = doc.createElement("geographicflag")
geographicflag.appendChild(geographicflag2)
spatialrefsys.appendChild(geographicflag)
# Create the <transparencyLevelInt> element
transparencyLevelInt = doc.createElement("transparencyLevelInt")
transparency2 = doc.createTextNode("255")
transparencyLevelInt.appendChild(transparency2)
maplayer.appendChild(transparencyLevelInt)
# Create the <customproperties> element
customproperties = doc.createElement("customproperties")
maplayer.appendChild(customproperties)
# Create the <provider> element
provider = doc.createElement("provider")
provider.setAttribute("encoding", "System")
ogr = doc.createTextNode("ogr")
provider.appendChild(ogr)
maplayer.appendChild(provider)
# Create the <singlesymbol> element
singlesymbol = doc.createElement("singlesymbol")
maplayer.appendChild(singlesymbol)
# Create the <symbol> element
symbol = doc.createElement("symbol")
singlesymbol.appendChild(symbol)
# Create the <lowervalue> element
lowervalue = doc.createElement("lowervalue")
symbol.appendChild(lowervalue)
# Create the <uppervalue> element
uppervalue = doc.createElement("uppervalue")
symbol.appendChild(uppervalue)
# Create the <label> element
label = doc.createElement("label")
symbol.appendChild(label)
# Create the <rotationclassificationfieldname> element
rotationclassificationfieldname = doc.createElement("rotationclassificationfieldname")
symbol.appendChild(rotationclassificationfieldname)
# Create the <scaleclassificationfieldname> element
scaleclassificationfieldname = doc.createElement("scaleclassificationfieldname")
symbol.appendChild(scaleclassificationfieldname)
# Create the <symbolfieldname> element
symbolfieldname = doc.createElement("symbolfieldname")
symbol.appendChild(symbolfieldname)
# Create the <outlinecolor> element
outlinecolor = doc.createElement("outlinecolor")
outlinecolor.setAttribute("red", "88")
outlinecolor.setAttribute("blue", "99")
outlinecolor.setAttribute("green", "37")
symbol.appendChild(outlinecolor)
# Create the <outlinestyle> element
outlinestyle = doc.createElement("outlinestyle")
outline = doc.createTextNode("SolidLine")
outlinestyle.appendChild(outline)
symbol.appendChild(outlinestyle)
# Create the <outlinewidth> element
outlinewidth = doc.createElement("outlinewidth")
width = doc.createTextNode("0.26")
outlinewidth.appendChild(width)
symbol.appendChild(outlinewidth)
# Create the <fillcolor> element
fillcolor = doc.createElement("fillcolor")
fillcolor.setAttribute("red", "90")
fillcolor.setAttribute("blue", "210")
fillcolor.setAttribute("green", "229")
symbol.appendChild(fillcolor)
# Create the <fillpattern> element
fillpattern = doc.createElement("fillpattern")
fill = doc.createTextNode("SolidPattern")
fillpattern.appendChild(fill)
symbol.appendChild(fillpattern)
# Create the <texturepath> element
texturepath = doc.createElement("texturepath")
texturepath.setAttribute("null", "1")
symbol.appendChild(texturepath)
map_canvas()
legend_func()
project_layers()
# Write to qgis file
try:
xml.dom.ext.PrettyPrint(doc, f)
finally:
f.close()
print 'Done'
The xml.dom.ext module was never added to the Python standard library.
It was only ever part of the PyXML distribution, but that has not seen any updates in years and I doubt it'll still work on Python 2.6.
Instead, just call the minidom .toprettyxml() method on your document to pretty print the output, then write that data out to the file:
f.write(doc.toprettyxml())
I'm trying to create a new dicom image from a standard-sized (512 x 512 or 256 x 256) numpy array.
import dicom, dicom.UID
from dicom.dataset import Dataset, FileDataset
def write_dicom(pixel_array,filename):
file_meta = Dataset()
ds = FileDataset(filename, {},file_meta = file_meta,preamble="\0"*128)
ds.PixelData = pixel_array.tostring()
ds.save_as(filename)
return
if __name__ == "__main__":
import numpy as np
pixel_array = np.tile(np.arange(256).reshape(16,16), (16,16)) * 4
write_dicom(pixel_array,'pretty.dcm')
2020 update :)
None of these answers worked for me. This is what I ended up with to save a valid monochrome 16bpp MR slice which is correctly displayed at least in Slicer, Radiant and MicroDicom:
import pydicom
from pydicom.dataset import Dataset, FileDataset
from pydicom.uid import ExplicitVRLittleEndian
import pydicom._storage_sopclass_uids
image2d = image2d.astype(np.uint16)
print("Setting file meta information...")
# Populate required values for file meta information
meta = pydicom.Dataset()
meta.MediaStorageSOPClassUID = pydicom._storage_sopclass_uids.MRImageStorage
meta.MediaStorageSOPInstanceUID = pydicom.uid.generate_uid()
meta.TransferSyntaxUID = pydicom.uid.ExplicitVRLittleEndian
ds = Dataset()
ds.file_meta = meta
ds.is_little_endian = True
ds.is_implicit_VR = False
ds.SOPClassUID = pydicom._storage_sopclass_uids.MRImageStorage
ds.PatientName = "Test^Firstname"
ds.PatientID = "123456"
ds.Modality = "MR"
ds.SeriesInstanceUID = pydicom.uid.generate_uid()
ds.StudyInstanceUID = pydicom.uid.generate_uid()
ds.FrameOfReferenceUID = pydicom.uid.generate_uid()
ds.BitsStored = 16
ds.BitsAllocated = 16
ds.SamplesPerPixel = 1
ds.HighBit = 15
ds.ImagesInAcquisition = "1"
ds.Rows = image2d.shape[0]
ds.Columns = image2d.shape[1]
ds.InstanceNumber = 1
ds.ImagePositionPatient = r"0\0\1"
ds.ImageOrientationPatient = r"1\0\0\0\-1\0"
ds.ImageType = r"ORIGINAL\PRIMARY\AXIAL"
ds.RescaleIntercept = "0"
ds.RescaleSlope = "1"
ds.PixelSpacing = r"1\1"
ds.PhotometricInterpretation = "MONOCHROME2"
ds.PixelRepresentation = 1
pydicom.dataset.validate_file_meta(ds.file_meta, enforce_standard=True)
print("Setting pixel data...")
ds.PixelData = image2d.tobytes()
ds.save_as(r"out.dcm")
Note the following:
Going through FileDataset constructor as PyDicom docs suggest was failing to create a valid header for me
validate_file_meta will create some missing elements in header for you (version)
You need to specify endianness and explicit/implicit VR twice :/
This method will allow you to create a valid volume as well as long as you update ImagePositionPatient and InstanceNumber for each slice accordingly
Make sure your numpy array is cast to data format that has same number of bits as your BitsStored
Here is a functional version of the code I needed to write. It will write a 16-bit grayscale DICOM image from a given 2D array of pixels. According to the DICOM standard, the UIDs should be unique for each image and series, which this code doesn't worry about, because I don't know what the UIDs actually do. If anyone else does, I'll be happy to add it in.
import dicom, dicom.UID
from dicom.dataset import Dataset, FileDataset
import numpy as np
import datetime, time
def write_dicom(pixel_array,filename):
"""
INPUTS:
pixel_array: 2D numpy ndarray. If pixel_array is larger than 2D, errors.
filename: string name for the output file.
"""
## This code block was taken from the output of a MATLAB secondary
## capture. I do not know what the long dotted UIDs mean, but
## this code works.
file_meta = Dataset()
file_meta.MediaStorageSOPClassUID = 'Secondary Capture Image Storage'
file_meta.MediaStorageSOPInstanceUID = '1.3.6.1.4.1.9590.100.1.1.111165684411017669021768385720736873780'
file_meta.ImplementationClassUID = '1.3.6.1.4.1.9590.100.1.0.100.4.0'
ds = FileDataset(filename, {},file_meta = file_meta,preamble="\0"*128)
ds.Modality = 'WSD'
ds.ContentDate = str(datetime.date.today()).replace('-','')
ds.ContentTime = str(time.time()) #milliseconds since the epoch
ds.StudyInstanceUID = '1.3.6.1.4.1.9590.100.1.1.124313977412360175234271287472804872093'
ds.SeriesInstanceUID = '1.3.6.1.4.1.9590.100.1.1.369231118011061003403421859172643143649'
ds.SOPInstanceUID = '1.3.6.1.4.1.9590.100.1.1.111165684411017669021768385720736873780'
ds.SOPClassUID = 'Secondary Capture Image Storage'
ds.SecondaryCaptureDeviceManufctur = 'Python 2.7.3'
## These are the necessary imaging components of the FileDataset object.
ds.SamplesPerPixel = 1
ds.PhotometricInterpretation = "MONOCHROME2"
ds.PixelRepresentation = 0
ds.HighBit = 15
ds.BitsStored = 16
ds.BitsAllocated = 16
ds.SmallestImagePixelValue = '\\x00\\x00'
ds.LargestImagePixelValue = '\\xff\\xff'
ds.Columns = pixel_array.shape[0]
ds.Rows = pixel_array.shape[1]
if pixel_array.dtype != np.uint16:
pixel_array = pixel_array.astype(np.uint16)
ds.PixelData = pixel_array.tostring()
ds.save_as(filename)
return
if __name__ == "__main__":
# pixel_array = np.arange(256*256).reshape(256,256)
# pixel_array = np.tile(np.arange(256).reshape(16,16),(16,16))
x = np.arange(16).reshape(16,1)
pixel_array = (x + x.T) * 32
pixel_array = np.tile(pixel_array,(16,16))
write_dicom(pixel_array,'pretty.dcm')
I was able to reduce #Corvin's great answer even more. Here is a minimalist code example allowing one to save a (dummy) 3D numpy array to a valid DICOM image that can be opened with Amide:
#!/usr/bin/python3
import numpy
import pydicom
import pydicom._storage_sopclass_uids
# dummy image
image = numpy.random.randint(2**16, size=(512, 512, 512), dtype=numpy.uint16)
# metadata
fileMeta = pydicom.Dataset()
fileMeta.MediaStorageSOPClassUID = pydicom._storage_sopclass_uids.CTImageStorage
fileMeta.MediaStorageSOPInstanceUID = pydicom.uid.generate_uid()
fileMeta.TransferSyntaxUID = pydicom.uid.ExplicitVRLittleEndian
# dataset
ds = pydicom.Dataset()
ds.file_meta = fileMeta
ds.Rows = image.shape[0]
ds.Columns = image.shape[1]
ds.NumberOfFrames = image.shape[2]
ds.PixelSpacing = [1, 1] # in mm
ds.SliceThickness = 1 # in mm
ds.BitsAllocated = 16
ds.PixelRepresentation = 1
ds.PixelData = image.tobytes()
# save
ds.save_as('image.dcm', write_like_original=False)
As one might observe, a lot of fields are missing if the output image.dcm file is passed to dciodvfy. The filling of these fields are left to the reader ;)
The above example works but causes many tools to complain about the DICOMs and they cannot even be read at all using itk/SimpleITK as a stack. The best way I have found for making DICOMs from numpy is by using the SimpleITK tools and generating the DICOMs slice-by-slice. A basic example (https://github.com/zivy/SimpleITK/blob/8e94451e4c0e90bcc6a1ffdd7bc3d56c81f58d80/Examples/DicomSeriesReadModifyWrite/DicomSeriesReadModifySeriesWrite.py) shows how to load in a stack, perform a transformation and then resave the files, but this can easily be modified by using the
import SimpleITK as sitk
filtered_image = sitk.GetImageFromArray(my_numpy_array)
The number of tags ultimately in output image is quite large and so manually creating all of them is tedious. Additionally SimpleITK supports 8, 16, 32-bit images as well as RGB so it is much easier than making them in pydicom.
(0008, 0008) Image Type CS: ['DERIVED', 'SECONDARY']
(0008, 0016) SOP Class UID UI: Secondary Capture Image Storage
(0008, 0018) SOP Instance UID UI: 1.2.826.0.1.3680043.2.1125.1.35596048796922805578234000521866725
(0008, 0020) Study Date DA: '20170803'
(0008, 0021) Series Date DA: '20170803'
(0008, 0023) Content Date DA: 0
(0008, 0030) Study Time TM: '080429.171808'
(0008, 0031) Series Time TM: '080429'
(0008, 0033) Content Time TM: 0
(0008, 0050) Accession Number SH: ''
(0008, 0060) Modality CS: 'OT'
(0008, 0064) Conversion Type CS: 'WSD'
(0008, 0090) Referring Physician's Name PN: ''
(0010, 0010) Patient's Name PN: ''
(0010, 0020) Patient ID LO: ''
(0010, 0030) Patient's Birth Date DA: ''
(0010, 0040) Patient's Sex CS: ''
(0018, 2010) Nominal Scanned Pixel Spacing DS: ['1', '3']
(0020, 000d) Study Instance UID UI: 1.2.826.0.1.3680043.2.1125.1.33389357207068897066210100430826006
(0020, 000e) Series Instance UID UI: 1.2.826.0.1.3680043.2.1125.1.51488923827429438625199681257282809
(0020, 0010) Study ID SH: ''
(0020, 0011) Series Number IS: ''
(0020, 0013) Instance Number IS: ''
(0020, 0020) Patient Orientation CS: ''
(0020, 0052) Frame of Reference UID UI: 1.2.826.0.1.3680043.2.1125.1.35696880630664441938326682384062489
(0028, 0002) Samples per Pixel US: 1
(0028, 0004) Photometric Interpretation CS: 'MONOCHROME2'
(0028, 0010) Rows US: 40
(0028, 0011) Columns US: 50
(0028, 0100) Bits Allocated US: 32
(0028, 0101) Bits Stored US: 32
(0028, 0102) High Bit US: 31
(0028, 0103) Pixel Representation US: 1
(0028, 1052) Rescale Intercept DS: "0"
(0028, 1053) Rescale Slope DS: "1"
(0028, 1054) Rescale Type LO: 'US'
(7fe0, 0010) Pixel Data OW: Array of 8000 bytes
Corvin's 2020 update almost worked for me.
The meta was still not written to the file, so when reading it the following exception was raised:
pydicom.errors.InvalidDicomError: File is missing DICOM File Meta Information header or the 'DICM' prefix is missing from the header.
In order to fix this and write the meta into the dicom file, I needed to add enforce_standard=True to the save_as() call:
ds.save_as(filename=out_filename, enforce_standard=True)
DICOM is a really complicated format. There are many dialects, and compatibilty is rather a question of luck. You could alternatively try nibabel, maybe its dialect is more appealing to RadiAnt or MicroDicom.
In general, I'd recommend using Nifti-format whenever possible. Its standard is much more concise, and incompatibilities are rare. nibabel also supports this.
One working config for those who need it and one question.
Question is in another thread Create a Dicom from multiple jpg images
What worked for me was greyscale without compression. Every attempt at compression fails miserably I don't know why:
# Populate required values for file meta information
meta = pydicom.Dataset()
meta.TransferSyntaxUID = pydicom.uid.ExplicitVRLittleEndian
meta.MediaStorageSOPClassUID = pydicom._storage_sopclass_uids.MRImageStorage
meta.MediaStorageSOPInstanceUID = pydicom.uid.generate_uid()
# build dataset
ds = Dataset()
ds.file_meta = meta
ds.fix_meta_info()
# unknown options
ds.is_little_endian = True
ds.is_implicit_VR = False
ds.SOPClassUID = pydicom._storage_sopclass_uids.MRImageStorage
ds.SeriesInstanceUID = pydicom.uid.generate_uid()
ds.StudyInstanceUID = pydicom.uid.generate_uid()
ds.FrameOfReferenceUID = pydicom.uid.generate_uid()
ds.BitsStored = 16
ds.BitsAllocated = 16
ds.SamplesPerPixel = 1
ds.HighBit = 15
ds.ImagesInAcquisition = "1"
ds.InstanceNumber = 1
ds.ImagePositionPatient = r"0\0\1"
ds.ImageOrientationPatient = r"1\0\0\0\-1\0"
ds.ImageType = r"ORIGINAL\PRIMARY\AXIAL"
ds.RescaleIntercept = "0"
ds.RescaleSlope = "1"
ds.PixelRepresentation = 1
# Case options
ds.PatientName = "Anonymous"
ds.PatientID = "123456"
ds.Modality = "MR"
ds.StudyDate = '20200225'
ds.ContentDate = '20200225'
# convert image to grayscale
img = Image.open(filename).convert('L')
img.save(filename)
# open image, decode and ensure_even stream
with open(filename, 'rb') as f:
arr = decode(f)
def ensure_even(stream):
# Very important for some viewers
if len(stream) % 2:
return stream + b"\x00"
return stream
# required for pixel handler
ds.BitsStored = 8
ds.BitsAllocated = 8
ds.HighBit = 7
ds.PixelRepresentation = 0
# grayscale without compression WORKS
ds.PhotometricInterpretation = "MONOCHROME2"
ds.SamplesPerPixel = 1 # 1 color = 1 sample per pixel
ds.file_meta.TransferSyntaxUID = pydicom.uid.ExplicitVRLittleEndian
ds.PixelData = ensure_even(arr.tobytes())
# JPEGBaseline compressed DOES NOT WORK
# ds.PixelData = encapsulate([ensure_even(arr.tobytes())])
# ds.PhotometricInterpretation = "YBR_FULL"
# ds.SamplesPerPixel = 3 # 3 colors = 3 sampleperpixel
# ds.file_meta.TransferSyntaxUID = pydicom.uid.JPEGBaseline
# ds.compress(pydicom.uid.JPEGBaseline)
# JPEGExtended compressed DOES NOT WORK
# ds.PixelData = encapsulate([ensure_even(arr.tobytes())])
# ds.PhotometricInterpretation = "YBR_FULL_422"
# ds.SamplesPerPixel = 3 # 3 colors = 3 sampleperpixel
# ds.file_meta.TransferSyntaxUID = pydicom.uid.JPEGExtended
# ds.compress(pydicom.uid.JPEGExtended)
# JPEG2000 compressed DOES NOT WORK
# ds.PhotometricInterpretation = "RGB"
# ds.SamplesPerPixel = 3 # 3 colors = 3 sampleperpixel
# ds.file_meta.TransferSyntaxUID = pydicom.uid.JPEG2000
# ds.PixelData = encapsulate([ensure_even(arr.tobytes())])
# ds.compress(pydicom.uid.JPEG2000)
# Image shape
ds['PixelData'].is_undefined_length = False
array_shape = arr.shape
ds.Rows = array_shape[0]
ds.Columns = array_shape[1]
# validate and save
pydicom.dataset.validate_file_meta(ds.file_meta, enforce_standard=True)
new_filename = filename.replace('.jpg', name + '.dcm')
ds.save_as(new_filename, write_like_original=False)
For a 3D CT scan, you can use the following code
def vol_to_dicom_for_ct(path_img_ct, patient_name, patient_id, path_dicom):
"""
Converts a .nrrd/.mha/.nifti file into its .dcm files
Params
------
path_img_ct: str, the path of the .nrrd/.mha/.nifti file
patient_name: str
patient_id: str
path_dicom: str, the final output directory
Note: Verify the output with dciodvfy
- Ref 1: https://www.dclunie.com/dicom3tools/workinprogress/index.html
- Ref 2: https://manpages.debian.org/unstable/dicom3tools/dciodvfy.1.en.html
"""
try:
import sys
import copy
import random
import shutil
import subprocess
import numpy as np
if Path(path_img_ct).exists():
try:
import pydicom
import pydicom._storage_sopclass_uids
except:
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'pydicom'])
import pydicom
try:
import SimpleITK as sitk
except:
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'SimpleITK']) # 2.1.1
import SimpleITK as sitk
try:
import matplotlib.pyplot as plt
except:
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', 'matplotlib']) # 2.1.1
import matplotlib.pyplot as plt
# Step 0 - Create save directory
if Path(path_dicom).exists():
shutil.rmtree(path_dicom)
Path(path_dicom).mkdir(exist_ok=True, parents=True)
# Step 1 - Get volume params
img_ct = sitk.ReadImage(str(path_img_ct))
img_spacing = tuple(img_ct.GetSpacing())
img_origin = tuple(img_ct.GetOrigin()) # --> dicom.ImagePositionPatient
img_array = sitk.GetArrayFromImage(img_ct).astype(np.int16) # [D,H,W]
# Step 2 - Create dicom dataset
ds = pydicom.dataset.Dataset()
ds.FrameOfReferenceUID = pydicom.uid.generate_uid() # this will stay the same for all .dcm files of a volume
# Step 2.1 - Modality details
ds.SOPClassUID = pydicom._storage_sopclass_uids.CTImageStorage
ds.Modality = 'CT'
ds.ImageType = ['ORIGINAL', 'PRIMARY', 'AXIAL']
# Step 2.2 - Image Details
ds.PixelSpacing = [float(img_spacing[0]), float(img_spacing[1])]
ds.SliceThickness = str(img_spacing[-1])
ds.Rows = img_array.shape[1]
ds.Columns = img_array.shape[2]
ds.PatientPosition = 'HFS'
ds.ImageOrientationPatient = [1, 0, 0, 0, 1, 0]
ds.PositionReferenceIndicator = 'SN'
ds.SamplesPerPixel = 1
ds.PhotometricInterpretation = 'MONOCHROME2'
ds.BitsAllocated = 16
ds.BitsStored = 16
ds.HighBit = 15
ds.PixelRepresentation = 1
ds.RescaleIntercept = "0.0"
ds.RescaleSlope = "1.0"
ds.RescaleType = 'HU'
# Step 3.1 - Metadata
fileMeta = pydicom.Dataset()
fileMeta.MediaStorageSOPClassUID = pydicom._storage_sopclass_uids.CTImageStorage
fileMeta.MediaStorageSOPInstanceUID = pydicom.uid.generate_uid() # this will change for each .dcm file of a volume
fileMeta.TransferSyntaxUID = pydicom.uid.ExplicitVRLittleEndian
ds.file_meta = fileMeta
# Step 3.2 - Include study details
ds.StudyInstanceUID = pydicom.uid.generate_uid()
ds.StudyDescription = ''
ds.StudyDate = '19000101' # needed to create DICOMDIR
ds.StudyID = str(random.randint(0,1000)) # needed to create DICOMDIR
# Step 3.3 - Include series details
ds.SeriesInstanceUID = pydicom.uid.generate_uid()
ds.SeriesDescription = ''
ds.SeriesNumber = str(random.randint(0,1000)) # needed to create DICOMDIR
# Step 3.4 - Include patient details
ds.PatientName = patient_name
ds.PatientID = patient_id
# Step 3.5 - Manufacturer details
ds.Manufacturer = 'MICCAI2015'
ds.ReferringPhysicianName = 'Mody' # needed for identification in RayStation
ds.ManufacturerModelName = 'test_offsite'
# Step 4 - Make slices
for slice_id in range(img_array.shape[0]):
# Step 4.1 - Slice identifier
random_uuid = pydicom.uid.generate_uid()
ds.file_meta.MediaStorageSOPInstanceUID = random_uuid
ds.SOPInstanceUID = random_uuid
ds.InstanceNumber = str(slice_id+1)
vol_origin_tmp = list(copy.deepcopy(img_origin))
vol_origin_tmp[-1] += img_spacing[-1]*slice_id
ds.ImagePositionPatient = vol_origin_tmp
# Step 4.2 - Slice data
img_slice = img_array[slice_id,:,:]
# plt.imshow(img_slice); plt.savefig(str(Path(path_dicom, '{}.png'.format(slice_id)))); plt.close()
ds.PixelData = img_slice.tobytes()
save_path = Path(path_dicom).joinpath(str(ds.file_meta.MediaStorageSOPInstanceUID) + '.dcm')
ds.save_as(str(save_path), write_like_original=False)
return ds.StudyInstanceUID, ds.SeriesInstanceUID
else:
print (' - [ERROR][vol_to_dicom_for_ct()] Error in path: path_img_ct: ', path_img_ct)
return None, None
except:
traceback.print_exc()