Related
As shown in the code posted below in section DecoupleGridCellsProfilerLoopsPool, the run() is called as much times as the contents of the self.__listOfLoopDecouplers and it works as it supposed to be, i mean the parallelization is working duly.
as shown in the same section, DecoupleGridCellsProfilerLoopsPool.pool.map returns results and i populate some lists,lets discuss the list names self.__iterablesOfZeroCoverageCell it contains number of objects of type gridCellInnerLoopsIteratorsForZeroCoverageModel.
After that, i created the pool ZeroCoverageCellsProcessingPool with the code as posted below as well.
The problem i am facing, is the parallelized code in ZeroCoverageCellsProcessingPool is very slow and the visulisation of the cpu tasks shows that there are no processes work in parallel as shown in the video contained in url posted below.
i was suspicious about the pickling issues related to when parallelizing the code in ZeroCoverageCellsProcessingPool,so i removed the enitre body of the run() in ZeroCoverageCellsProcessingPool. however, it shows no change in the behaviour of the parallelized code.
also the url posted below shown how the parallelized methoth of ZeroCoverageCellsProcessingPool behaves.
given the code posted below, please let me know why the parallelization does not work for code in ZeroCoverageCellsProcessingPool
output url:please click the link
output url
DecoupleGridCellsProfilerLoopsPool
def postTask(self):
self.__postTaskStartTime = time.time()
with Pool(processes=int(config['MULTIPROCESSING']['proceses_count'])) as DecoupleGridCellsProfilerLoopsPool.pool:
self.__chunkSize = PoolUtils.getChunkSize(lst=self.__listOfLoopDecouplers,cpuCount=int(config['MULTIPROCESSING']['cpu_count']))
logger.info(f"DecoupleGridCellsProfilerLoopsPool.self.__chunkSize(task per processor):{self.__chunkSize}")
for res in DecoupleGridCellsProfilerLoopsPool.pool.map(self.run,self.__listOfLoopDecouplers,chunksize=self.__chunkSize):
if res[0] is not None and res[1] is None and res[2] is None:
self.__iterablesOfNoneZeroCoverageCell.append(res[0])
elif res[1] is not None and res[0] is None and res[2] is None:
self.__iterablesOfZeroCoverageCell.append(res[1])
elif res[2] is not None and res[0] is None and res[1] is None:
self.__iterablesOfNoDataCells.append(res[2])
else:
raise Exception (f"WTF.")
DecoupleGridCellsProfilerLoopsPool.pool.join()
assert len(self.__iterablesOfNoneZeroCoverageCell)+len(self.__iterablesOfZeroCoverageCell)+len(self.__iterablesOfNoDataCells) == len(self.__listOfLoopDecouplers)
zeroCoverageCellsProcessingPool = ZeroCoverageCellsProcessingPool(self.__devModeForWSAWANTIVer2,self.__iterablesOfZeroCoverageCell)
zeroCoverageCellsProcessingPool.postTask()
def run(self,param:LoopDecoupler):
row = param.getRowValue()
col = param.getColValue()
elevationsTIFFWindowedSegmentContents = param.getElevationsTIFFWindowedSegment()
verticalStep = param.getVericalStep()
horizontalStep = param.getHorizontalStep()
mainTIFFImageDatasetContents = param.getMainTIFFImageDatasetContents()
NDVIsTIFFWindowedSegmentContentsInEPSG25832 = param.getNDVIsTIFFWindowedSegmentContentsInEPSG25832()
URLOrFilePathForElevationsTIFFDatasetInEPSG25832 = param.getURLOrFilePathForElevationsTIFFDatasetInEPSG25832()
threshold = param.getThreshold()
rowsCnt = 0
colsCnt = 0
pixelsValuesSatisfyThresholdInTIFFImageDatasetCnt = 0
pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt = int(config['window']['width']) * int(config['window']['height'])
pixelsWithNoDataValueInTIFFImageDatasetCnt = int(config['window']['width']) * int(config['window']['height'])
_pixelsValuesSatisfyThresholdInNoneZeroCoverageCell = []
_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell = []
_pixelsValuesInNoDataCell = []
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel = None
gridCellInnerLoopsIteratorsForZeroCoverageModel = None
gridCellInnerLoopsIteratorsForNoDataCellsModel = None
for x in range(row,row + verticalStep):
if rowsCnt == verticalStep:
rowsCnt = 0
for y in range(col,col + horizontalStep):
if colsCnt == horizontalStep:
colsCnt = 0
pixelValue = mainTIFFImageDatasetContents[0][x][y]
# windowIOUtils.writeContentsToFile(windowIOUtils.getPathToOutputDir()+"/"+config['window']['file_name']+".{0}".format(config['window']['file_extension']), "pixelValue:{0}\n".format(pixelValue))
if pixelValue >= float(threshold):
pixelsValuesSatisfyThresholdInTIFFImageDatasetCnt+=1
_pixelsValuesSatisfyThresholdInNoneZeroCoverageCell.append(elevationsTIFFWindowedSegmentContents[0][rowsCnt][colsCnt])
elif ((pixelValue < float(threshold)) and (pixelValue > float(config['TIFF']['no_data_value']))):
pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt-=1
_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell.append(elevationsTIFFWindowedSegmentContents[0][rowsCnt][colsCnt])
elif (pixelValue <= float(config['TIFF']['no_data_value'])):
pixelsWithNoDataValueInTIFFImageDatasetCnt-=1
_pixelsValuesInNoDataCell.append(elevationsTIFFWindowedSegmentContents[0][rowsCnt][colsCnt])
else:
raise Exception ("WTF.Exception: unhandled condition for pixel value: {0}".format(pixelValue))
# _pixelCoordinatesInWindow.append([x,y])
colsCnt+=1
rowsCnt+=1
'''Grid-cell classfication'''
if (pixelsValuesSatisfyThresholdInTIFFImageDatasetCnt > 0):
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel = GridCellInnerLoopsIteratorsForNoneZeroCoverageModel()
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setRowValue(row)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setColValue(col)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setVericalStep(verticalStep)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setHorizontalStep(horizontalStep)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setMainTIFFImageDatasetContents(mainTIFFImageDatasetContents)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setNDVIsTIFFWindowedSegmentContentsInEPSG25832(NDVIsTIFFWindowedSegmentContentsInEPSG25832)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setURLOrFilePathForElevationsTIFFDatasetInEPSG25832(URLOrFilePathForElevationsTIFFDatasetInEPSG25832)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setPixelsValuesSatisfyThresholdInTIFFImageDatasetCnt(pixelsValuesSatisfyThresholdInTIFFImageDatasetCnt)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setPixelsValuesSatisfyThresholdInNoneZeroCoverageCell(_pixelsValuesSatisfyThresholdInNoneZeroCoverageCell)
elif (pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt < (int(config['window']['width']) * int(config['window']['height'])) and pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt >= 0):
gridCellInnerLoopsIteratorsForZeroCoverageModel = GridCellInnerLoopsIteratorsForZeroCoverageModel()
gridCellInnerLoopsIteratorsForZeroCoverageModel.setRowValue(row)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setColValue(col)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setVericalStep(verticalStep)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setHorizontalStep(horizontalStep)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setMainTIFFImageDatasetContents(mainTIFFImageDatasetContents)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setNDVIsTIFFWindowedSegmentContentsInEPSG25832(NDVIsTIFFWindowedSegmentContentsInEPSG25832)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setURLOrFilePathForElevationsTIFFDatasetInEPSG25832(URLOrFilePathForElevationsTIFFDatasetInEPSG25832)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt(pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setPixelsWithNoDataValueInTIFFImageDatasetCnt(pixelsWithNoDataValueInTIFFImageDatasetCnt)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setPixelsValuesDoNotSatisfyThresholdInZeroCoverageCell(_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell)
elif (pixelsWithNoDataValueInTIFFImageDatasetCnt == 0):
gridCellInnerLoopsIteratorsForNoDataCellsModel = GridCellInnerLoopsIteratorsForNoDataCellsModel()
gridCellInnerLoopsIteratorsForNoDataCellsModel.setRowValue(row)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setColValue(col)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setVericalStep(verticalStep)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setHorizontalStep(horizontalStep)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setMainTIFFImageDatasetContents(mainTIFFImageDatasetContents)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setNDVIsTIFFWindowedSegmentContentsInEPSG25832(NDVIsTIFFWindowedSegmentContentsInEPSG25832)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setURLOrFilePathForElevationsTIFFDatasetInEPSG25832(URLOrFilePathForElevationsTIFFDatasetInEPSG25832)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setPixelsWithNoDataValueInTIFFImageDatasetCnt(pixelsWithNoDataValueInTIFFImageDatasetCnt)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setPixelsValuesInNoDataCell(_pixelsValuesInNoDataCell)
if gridCellInnerLoopsIteratorsForZeroCoverageModel is not None:
gridCellInnerLoopsIteratorsForZeroCoverageModel.setPixelsWithNoDataValueInTIFFImageDatasetCnt(pixelsWithNoDataValueInTIFFImageDatasetCnt)
else:
raise Exception (f"WTF.")
return gridCellInnerLoopsIteratorsForNoneZeroCoverageModel,gridCellInnerLoopsIteratorsForZeroCoverageModel,gridCellInnerLoopsIteratorsForNoDataCellsModel
ZeroCoverageCellsProcessingPool:
def postTask(self):
self.__postTaskStartTime = time.time()
"""to collect results per each row
"""
resAllCellsForGridCellsClassifications = []
# NDVIs
resAllCellsForNDVITIFFDetailsForZeroCoverageCell = []
# area of coverage
resAllCellsForAreaOfCoverageForZeroCoverageCell = []
# interception
resAllCellsForInterceptionForZeroCoverageCell = []
# fourCornersOfWindowInEPSG25832
resAllCellsForFourCornersOfWindowInEPSG25832ZeroCoverageCell = []
# outFromEPSG25832ToEPSG4326-lists
resAllCellsForOutFromEPSG25832ToEPSG4326ForZeroCoverageCells = []
# fourCornersOfWindowsAsGeoJSON
resAllCellsForFourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell = []
# calculatedCenterPointInEPSG25832
resAllCellsForCalculatedCenterPointInEPSG25832ForZeroCoverageCell = []
# centerPointsOfWindowInImageCoordinatesSystem
resAllCellsForCenterPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell = []
# pixelValuesOfCenterPoints
resAllCellsForPixelValuesOfCenterPointsForZeroCoverageCell = []
# centerPointOfKeyWindowAsGeoJSONInEPSG4326
resAllCellsForCenterPointOfKeyWindowAsGeoJSONInEPSG4326ForZeroCoverageCell = []
# centerPointInEPSG4326
resAllCellsForCenterPointInEPSG4326ForZeroCoveringCell = []
# average heights
resAllCellsForAverageHeightsForZeroCoverageCell = []
# pixels values
resAllCellsForPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCell = []
# area Of Coverage
resAllCellsForAreaOfCoverageForZeroCoverageCell = []
resAllCellsForPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt = []
noneKeyWindowCnt=0
# center points as string
centerPointsAsStringForZeroCoverageCell = ""
with Pool(processes=int(config['MULTIPROCESSING']['proceses_count'])) as ZeroCoverageCellsProcessingPool.pool:
self.__chunkSize = PoolUtils.getChunkSize(lst=self.__iterables,cpuCount=int(config['MULTIPROCESSING']['cpu_count']))
logger.info(f"ZeroCoverageCellsProcessingPool.self.__chunkSize(task per processor):{self.__chunkSize}")
for res in ZeroCoverageCellsProcessingPool.pool.map(func=self.run,iterable=self.__iterables,chunksize=self.__chunkSize):
resAllCellsForGridCellsClassifications.append(res[0])
# NDVIs
resAllCellsForNDVITIFFDetailsForZeroCoverageCell.append(res[1])
# area of coverage
resAllCellsForAreaOfCoverageForZeroCoverageCell.append(res[2])
# interception
resAllCellsForInterceptionForZeroCoverageCell.append(res[3])
# fourCornersOfWindowInEPSG25832
resAllCellsForFourCornersOfWindowInEPSG25832ZeroCoverageCell.append(res[4])
# outFromEPSG25832ToEPSG4326-lists
resAllCellsForOutFromEPSG25832ToEPSG4326ForZeroCoverageCells.append(res[5])
# fourCornersOfWindowsAsGeoJSONInEPSG4326
resAllCellsForFourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell.append(res[6])
# calculatedCenterPointInEPSG25832
resAllCellsForCalculatedCenterPointInEPSG25832ForZeroCoverageCell.append(res[7])
# centerPointsOfWindowInImageCoordinatesSystem
resAllCellsForCenterPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell.append(res[8])
# pixelValuesOfCenterPoints
resAllCellsForPixelValuesOfCenterPointsForZeroCoverageCell.append(res[9])
# centerPointInEPSG4326
resAllCellsForCenterPointInEPSG4326ForZeroCoveringCell.append(res[10])
# centerPointOfKeyWindowAsGeoJSONInEPSG4326
resAllCellsForCenterPointOfKeyWindowAsGeoJSONInEPSG4326ForZeroCoverageCell.append(res[11])
# average heights
resAllCellsForAverageHeightsForZeroCoverageCell.append(res[12])
# pixels values
resAllCellsForPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCell.append(res[13])
# pixelsValues cnt
resAllCellsForPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt.append(res[14])
noneKeyWindowCnt +=res[15]
# centerPoints-As-String
if (res[16] is not None):
centerPointsAsStringForZeroCoverageCell+=str(res[16])
assert noneKeyWindowCnt == len(self.__iterables)
ZeroCoverageCellsProcessingPool.pool.close()
ZeroCoverageCellsProcessingPool.pool.terminate()
ZeroCoverageCellsProcessingPool.pool.join()
return
def run(self,params:GridCellInnerLoopsIteratorsForZeroCoverageModel):
if params is not None:
logger.info(f"Processing zero coverage cell #(row{params.getRowValue()},col:{params.getColValue()})")
row = params.getRowValue()
col = params.getColValue()
mainTIFFImageDatasetContents = params.getMainTIFFImageDatasetContents()
NDVIsTIFFWindowedSegmentContentsInEPSG25832 = params.getNDVIsTIFFWindowedSegmentContentsInEPSG25832()
URLOrFilePathForElevationsTIFFDatasetInEPSG25832 = params.getURLOrFilePathForElevationsTIFFDatasetInEPSG25832()
datasetElevationsTIFFInEPSG25832 = rasterio.open(URLOrFilePathForElevationsTIFFDatasetInEPSG25832,'r')
_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell = params.getPixelsValuesDoNotSatisfyThresholdInZeroCoverageCell()
pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt = params.getPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt()
countOfNoDataCells = params.getPixelsWithNoDataValueInTIFFImageDatasetCnt()
outFromEPSG25832ToEPSG4326ForZeroCoverageCells = []
fourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell = []
ndviTIFFDetailsForZeroCoverageCell = NDVITIFFDetails(None,None,None).getNDVIValuePer10mX10m()
"""area of coverage per grid-cell"""
areaOfCoverageForZeroCoverageCell = None
""""interception"""
interceptionForZeroCoverageCell = None
CntOfNDVIsWithNanValueInZeroCoverageCell = 0
fourCornersOfWindowInEPSG25832ZeroCoverageCell = None
outFromEPSG25832ToEPSG4326ForZeroCoverageCells = []
fourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell = []
calculatedCenterPointInEPSG25832ForZeroCoverageCell = None
centerPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell = None
pixelValuesOfCenterPointsOfZeroCoverageCell = None
centerPointInEPSG4326ForZeroCoveringCell = None
centerPointOfKeyWindowAsGeoJSONInEPSG4326ForZeroCoverageCell = None
centerPointsAsStringForZeroCoverageCell = None
"""average heights"""
averageHeightsForZeroCoverageCell = None
gridCellClassifiedAs = GridCellClassifier.ZERO_COVERAGE_CELL.value
cntOfNoneKeyWindow = 1
ndviTIFFDetailsForZeroCoverageCell = NDVITIFFDetails(ulX=row//int(config['ndvi']['resolution_height']),ulY=col//int(config['ndvi']['resolution_width']),dataset=NDVIsTIFFWindowedSegmentContentsInEPSG25832).getNDVIValuePer10mX10m()
"""area of coverage per grid-cell"""
areaOfCoverageForZeroCoverageCell = round(AreaOfCoverageDetails(pixelsCount=(int(config['window']['width']) * int(config['window']['height'])) - (pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt + countOfNoDataCells)).getPercentageOfAreaOfCoverage(),2)
""""interception"""
if math.isnan(ndviTIFFDetailsForZeroCoverageCell):
# ndviTIFFDetailsForZeroCoverageCell = 0
CntOfNDVIsWithNanValueInZeroCoverageCell = 1
interceptionForZeroCoverageCell = config['sentinel_values']['interception']
else:
Indvi = INDVI()
Ic = Indvi.calcInterception(ndviTIFFDetailsForZeroCoverageCell)
Pc=areaOfCoverageForZeroCoverageCell,"""percentage of coverage"""
Pnc=float((int(config['window']['width'])*int(config['window']['height'])) - areaOfCoverageForZeroCoverageCell),"""percentage of non-coverage"""
Inc=float(config['interception']['noneCoverage']),"""interception of none-coverage"""
I=(float(Pc[0])*(Ic))+float((Pnc[0]*Inc[0]))
interceptionForZeroCoverageCell = round(I,2)
if I != 10 and I != float('nan'):
logger.error(f"ndviTIFFDetailsForZeroCoverageCell:{ndviTIFFDetailsForZeroCoverageCell}")
logger.error(f"I:{I}")
fourCornersOfWindowInEPSG25832ZeroCoverageCell = RasterIOPackageUtils.convertFourCornersOfWindowFromImageCoordinatesToCRSByCoordinatesOfCentersOfPixelsMethodFor(row,col,int(config['window']['height']),int(config['window']['width']),datasetElevationsTIFFInEPSG25832)
for i in range(0,len(fourCornersOfWindowInEPSG25832ZeroCoverageCell)):
# fourCornersOfKeyWindowInEPSG4326.append(RasterIOPackageUtils.convertCoordsToDestEPSGForDataset(fourCornersOfWindowInEPSG25832[i],datasetElevationsTIFFInEPSG25832,destEPSG=4326))
outFromEPSG25832ToEPSG4326ForZeroCoverageCells.append(OSGEOUtils.fromEPSG25832ToEPSG4326(fourCornersOfWindowInEPSG25832ZeroCoverageCell[i])) # resultant coords order is in form of lat,lon and it must be in lon,lat.thus, out[1]-lat out[0]-lon
"""fourCornersOfWindowsAsGeoJSONInEPSG4326"""
fourCornersOfWindowInEPSG4326 = []
for i in range(0,len(outFromEPSG25832ToEPSG4326ForZeroCoverageCells)):
fourCornersOfWindowInEPSG4326.append(([outFromEPSG25832ToEPSG4326ForZeroCoverageCells[i][1]],[outFromEPSG25832ToEPSG4326ForZeroCoverageCells[i][0]]))
fourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell.append(jsonUtils.buildFeatureCollectionAsGeoJSONForFourCornersOfKeyWindow(fourCornersOfWindowInEPSG4326[0],fourCornersOfWindowInEPSG4326[1],fourCornersOfWindowInEPSG4326[2],fourCornersOfWindowInEPSG4326[3],areaOfCoverageForZeroCoverageCell))
# debugIOUtils.writeContentsToFile(debugIOUtils.getPathToOutputDir()+"/"+"NDVIsPer10mX10mForKeyWindow"+config['window']['file_name']+".{0}".format(config['window']['file_extension']),"{0}\n".format(NDVIsPer10mX10mForKeyWindow))
"""
building geojson object for a point "center-point" to visualize it.
"""
calculatedCenterPointInEPSG25832ForZeroCoverageCell = MiscUtils.calculateCenterPointsGivenLLOfGridCell(fourCornersOfWindowInEPSG25832ZeroCoverageCell[1])#lower-left corner
centerPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell = RasterIOPackageUtils.convertFromCRSToImageCoordinatesSystemFor(calculatedCenterPointInEPSG25832ForZeroCoverageCell[0],calculatedCenterPointInEPSG25832ForZeroCoverageCell[1],datasetElevationsTIFFInEPSG25832)
pixelValuesOfCenterPointsOfZeroCoverageCell = mainTIFFImageDatasetContents[0][centerPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell[0]][centerPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell[1]]
centerPointInEPSG4326ForZeroCoveringCell = RasterIOPackageUtils.convertCoordsToDestEPSGForDataset(calculatedCenterPointInEPSG25832ForZeroCoverageCell,datasetElevationsTIFFInEPSG25832,destEPSG=4326)
centerPointOfKeyWindowAsGeoJSONInEPSG4326ForZeroCoverageCell = jsonUtils.buildGeoJSONForPointFor(centerPointInEPSG4326ForZeroCoveringCell)
"""average heights"""
averageHeightsForZeroCoverageCell = round(MiscUtils.getAverageFor(_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell),2)
assert len(_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell) > 0 and (len(_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell) <= (int(config['window']['width']) * int(config['window']['height'])) )
"""the following code block is for assertion only"""
if self.__devModeForWSAWANTIVer2 == config['DEVELOPMENT_MODE']['debug']:
assert pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt >= 0 and (pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt < (int(config['window']['width']) * int(config['window']['height'])) )
assert (pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt+countOfNoDataCells) == (int(config['window']['width']) * int(config['window']['height']))
print(f"profiling for gridCellClassifiedAs:{gridCellClassifiedAs}....>pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt:{pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt}")
print(f"profiling for gridCellClassifiedAs:{gridCellClassifiedAs}....>countOfNoDataCells:{countOfNoDataCells}")
pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt = (int(config['window']['width']) * int(config['window']['height'])) - (pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt + countOfNoDataCells)
assert pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt == 0, (f"WTF.")
print(f"profiling for gridCellClassifiedAs:{gridCellClassifiedAs}....>computed pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt:{pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt}")
print(f"\n")
centerPointAsTextInWKTInEPSG3857 = CoordinatesUtils.buildWKTPointFormatForSinglePointFor(calculatedCenterPointInEPSG25832ForZeroCoverageCell[0],calculatedCenterPointInEPSG25832ForZeroCoverageCell[1])
s = centerPointAsTextInWKTInEPSG3857.replace("POINT","")
s = s.replace("(","")
s = s.replace(")","")
s = s.strip()
s = s.split(" ")
centerPointsAsStringForZeroCoverageCell = s[0] + "\t" + s[1] + "\n"
centerPointsAsStringForZeroCoverageCell = centerPointsAsStringForZeroCoverageCell.replace('\'',"")
return gridCellClassifiedAs,ndviTIFFDetailsForZeroCoverageCell,areaOfCoverageForZeroCoverageCell,interceptionForZeroCoverageCell,fourCornersOfWindowInEPSG25832ZeroCoverageCell,outFromEPSG25832ToEPSG4326ForZeroCoverageCells,fourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell,calculatedCenterPointInEPSG25832ForZeroCoverageCell,centerPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell,pixelValuesOfCenterPointsOfZeroCoverageCell,centerPointInEPSG4326ForZeroCoveringCell,centerPointOfKeyWindowAsGeoJSONInEPSG4326ForZeroCoverageCell,averageHeightsForZeroCoverageCell,np.array(_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell).tolist(),pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt,cntOfNoneKeyWindow,centerPointsAsStringForZeroCoverageCell,CntOfNDVIsWithNanValueInZeroCoverageCell
I import technicals.py into bot.py and want to reuse the variable sl and tp from the class instance process_candles.
If a constant number is given to sl and tp in bot.py, the script is able to work. However, the desired result is to get variable sl and tp which is calculated in the class instance process_candles. from technicals.py.
snippet technicals.py as below:
df['PAIR'] = self.pair
decision = NONE
tp = 0
sl = 0
if c[-2]>o[-2]:
if ca[-1]>h[-2]+0.0010:
decision = BUY
tp = ca[-1]+0.010
sl = l[-2]-0.010
elif o[-2]>c[-2]:
if cb[-1]<l[-2]-0.0010:
decision = SELL
tp = cb[-1]-0.010
sl = h[-2]+0.010
else:
decision = NONE
snippet bot.py
def process_pairs(self):
trades_to_make = []
for pair in self.trade_pairs:
if self.timings[pair].ready == True:
self.log_message(f"Ready to trade {pair}")
techs = Technicals(self.settings[pair], self.api, pair, GRANULARITY, log=self.tech_log)
decision = techs.get_trade_decision(self.timings[pair].last_candle)
print ("process decision")
print (decision)
units = decision * self.settings[pair].units
#tp = "154"
#sl = "153"
if units != 0:
trades_to_make.append({'pair': pair, 'units': units,'take_profit':tp, 'stop_loss':sl})
Full script are as below:
technicals.py
import pandas as pd
import numpy as np
from defs import BUY, SELL, NONE
class Technicals():
def __init__(self, settings, api, pair, granularity, log=None):
self.settings = settings
self.log = log
self.api = api
self.pair = pair
self.granularity = granularity
def log_message(self, msg):
if self.log is not None:
self.log.logger.debug(msg)
def fetch_candles(self, row_count, candle_time):
status_code, df = self.api.fetch_candles(self.pair, count=row_count, granularity=self.granularity)
if df is None:
self.log_message(f"Error fetching candles for pair:{self.pair} {candle_time}, df None")
return None
elif df.iloc[-1].time != candle_time:
self.log_message(f"Error fetching candles for pair:{self.pair} {candle_time} vs {df.iloc[-1].time}")
return None
else:
return df
def process_candles(self, df):
open = df.mid_o
o = np.array(open,dtype='float')
#print (o)
high = df.mid_h
h = np.array(high,dtype='float')
#print (h)
low = df.mid_l
l = np.array(low,dtype='float')
#print (l)
close = df.mid_c
c = np.array(close,dtype='float')
print (c)
close_ask = df.ask_c
ca = np.array(close_ask,dtype='float')
print (ca)
close_bid = df.bid_c
cb = np.array(close_bid,dtype='float')
print (cb)
df['PAIR'] = self.pair
decision = NONE
tp = 0
sl = 0
if c[-2]>o[-2]:
if ca[-1]>h[-2]+0.0010:
decision = BUY
tp = ca[-1]+0.010
sl = l[-2]-0.010
elif o[-2]>c[-2]:
if cb[-1]<l[-2]-0.0010:
decision = SELL
tp = cb[-1]-0.010
sl = h[-2]+0.010
else:
decision = NONE
log_cols = ['time','volume','PAIR','bid_c','ask_c','mid_o','mid_h','mid_l','mid_c']
self.log_message(f"Processed_df\n{df[log_cols].tail(3)}")
self.log_message(f"Trade_decision:{decision}")
self.log_message("")
return decision
def get_trade_decision(self, candle_time):
max_rows = self.settings.long_ma + 2
self.log_message("")
self.log_message(f"get_trade_decision() pair:{self.pair} max_rows:{max_rows}")
df = self.fetch_candles(max_rows, candle_time)
print ("xxxx")
print (df)
if df is not None:
return self.process_candles(df)
print("get trade decision")
print(self.process_candles(df))
return NONE
bot.py
import pprint
import time
from settings import Settings
from log_wrapper import LogWrapper
from timing import Timing
from oanda_api import OandaAPI
from technicals import Technicals
from defs import NONE, BUY, SELL
from trade_manager import TradeManager
GRANULARITY = "M1"
SLEEP = 10.0
class TradingBot():
def __init__(self):
self.log = LogWrapper("Bot")
self.tech_log = LogWrapper("Technicals")
self.trade_log = LogWrapper("Trade")
self.trade_pairs = Settings.get_pairs()
self.settings = Settings.load_settings()
self.api = OandaAPI()
self.trade_manager = TradeManager(self.api, self.settings, self.trade_log)
self.timings = { p: Timing(self.api.last_complete_candle(p, GRANULARITY)) for p in self.trade_pairs }
self.log_message(f"Bot started with\n{pprint.pformat(self.settings)}")
self.log_message(f"Bot Timings\n{pprint.pformat(self.timings)}")
print (self.api)
def log_message(self, msg):
self.log.logger.debug(msg)
def update_timings(self):
for pair in self.trade_pairs:
current = self.api.last_complete_candle(pair, GRANULARITY)
self.timings[pair].ready = False
if current > self.timings[pair].last_candle:
self.timings[pair].ready = True
self.timings[pair].last_candle = current
self.log_message(f"{pair} new candle {current}")
def process_pairs(self):
trades_to_make = []
for pair in self.trade_pairs:
if self.timings[pair].ready == True:
self.log_message(f"Ready to trade {pair}")
techs = Technicals(self.settings[pair], self.api, pair, GRANULARITY, log=self.tech_log)
decision = techs.get_trade_decision(self.timings[pair].last_candle)
print ("process decision")
print (decision)
units = decision * self.settings[pair].units
#tp = "154"
#sl = "153"
if units != 0:
trades_to_make.append({'pair': pair, 'units': units,'take_profit':tp, 'stop_loss':sl})
if len(trades_to_make) > 0:
print("bot")
print(trades_to_make)
self.trade_manager.place_trades(trades_to_make)
def run(self):
while True:
self.update_timings()
self.process_pairs()
time.sleep(SLEEP)
if __name__ == "__main__":
b = TradingBot()
b.run()
defs.py
API_KEY = "xxxx"
ACCOUNT_ID = "xyz"
OANDA_URL = 'https://api-fxpractice.oanda.com/v3'
SECURE_HEADER = {
'Authorization': f'Bearer {API_KEY}',
'Content-Type': 'application/json'
}
BUY = 1
SELL = -1
NONE = 0
Instead of just returning the decision, also return the take profit and stop loss values:
return decision, tp, sl
Then you can unpack the tuple in process_pairs:
decision, tp, sl = techs.get_trade_decision(self.timings[pair].last_candle)
You can define your tp and sl as class variables of Technicals.
class Technicals(object):
tp: int = 0
sl: int = 0
and use them within Technicals as:
cls.tp = ... # if you are inside class-method
self.tp = ... # if you are inside instance-method
And in the TradingBot you can then simple import Technicals and use the class-Vars like:
tp = Technicals.tp # you can use the class
tp = techs.tp # or the instance you already have
I edit this post for your comments. Thank you :-)
The prev_fs_cell is the variable whose value can be nan or str. (ex. nan <-> "1,244,234" )
If prev_fs_cell is nan, I want not to process self._strat(self, curr_year), but it has an error...
## GLOBAL & API ###
STOCK_START="2015.01.01"
FS_START="2014.01.01"
END="2021.09.01"
SHORT=10
LONG=60
CURR_YEAR=2021
API_key=dart_config.API_key
DART=OpenDartReader(API_key)
account_nm_list=["유동자산","비유동자산","유동부채","비유동부채","자산총계","부채총계","매출액","영업이익","당기순이익"]
KOSPI_stock_code=stock.get_market_ticker_list(market="KOSPI")
class Strategy():
def __init__(self):
self.buy_signal=pd.DataFrame(columns=['open','unit'])
self.sell_signal = pd.DataFrame(columns=['open', 'unit'])
self.trade = pd.DataFrame(columns=['stock', 'cash'])
self.position=0
self.unit=1
self.cash=100000000 # 1억
def set_data(self, indicator_data, finance_data):
self.indicator_data=indicator_data
self.indicator_data.rename(columns={self.indicator_data.columns[0]:'date'}, inplace=True)
self.indicator_data = self.indicator_data.set_index('date')
self.indicator_data.index = pd.to_datetime(self.indicator_data.index, format="%Y-%m-%d")
self.fs_data=finance_data
self.fs_data.rename(columns={self.fs_data.columns[0]:'year'}, inplace=True)
self.fs_data = self.fs_data.set_index('year')
self.min_year=int(self.fs_data.index.min()) # str type
def _buy(self, row):
if (row['open']*self.unit) <= self.cash:
new_buy_row = pd.Series([row['open'], self.unit], index = self.buy_signal.columns, name=str(row.name))
self.buy_signal = self.buy_signal.append(new_buy_row)
self.position += self.unit
stock_amt = self.position * row['open']
self.cash -= row['open']*self.unit
new_trade_row = pd.Series([stock_amt, self.cash], index = self.trade.columns, name = str(row.name))
self.trade = self.trade.append(new_trade_row)
def _sell(self, row):
new_sell_row = pd.Series([row['open'], int(self.position/4)+1], index = self.sell_signal.columns, name=str(row.name))
self.sell_signal = self.sell_signal.append(new_sell_row)
self.position -= int(self.position/4)+1
stock_amt = self.position * row['open']
self.cash += row['open']*self.unit
new_trade_row = pd.Series([stock_amt, self.cash], index = self.trade.columns, name = str(row.name))
self.trade = self.trade.append(new_trade_row)
def _strat(self, row, curr_year):
fs = self.fs_data
prev_year = curr_year - 1
curr_rev = int(fs.loc[curr_year, '매출액'].replace(",",""))
prev_rev = int(fs.loc[prev_year, '매출액'].replace(",",""))
rev_growth=(curr_rev-prev_rev)/prev_rev
curr_ni = int(fs.loc[curr_year, '당기순이익'].replace(",",""))
prev_ni = int(fs.loc[prev_year, '당기순이익'].replace(",",""))
ni_growth=(curr_ni-prev_ni)/prev_ni
curr_asset = int(fs.loc[curr_year, '유동자산'].replace(",",""))
noncurr_asset = int(fs.loc[prev_year, "비유동자산"].replace(",",""))
curr_asset_rat = curr_asset / noncurr_asset
if (row.rsi<0.65) & (rev_growth>0.005) & (1.3< curr_asset_rat):# & (curr_asset_rat<2.3):
self._buy(row)
elif (row.Golden == False):
if ni_growth <= 0.001 :
if self.position:
self._sell(row)
# a=1
def run(self):
dates = self.indicator_data.index
fs = self.fs_data
#print(fs.index)
for date in dates:
curr_year = date.year
row = self.indicator_data.loc[date]
#print(curr_year, type(curr_year))
#pdb.set_trace()
try:
curr_fs_cell = fs.loc[curr_year].iloc[0].replace(",","")
try:
prev_fs_cell = fs.loc[curr_year-1].iloc[0].replace(",","")
except:
prev_fs_cell = None
except:
curr_fs_cell = None
if (curr_fs_cell == None) | (prev_fs_cell == None):
#print("fs data is empty")
continue
else:
#print(prev_fs_cell)
self._strat(row, curr_year)
for code in KOSPI_stock_code:
FS = load_data("FS_"+code)
indi = load_data("indicator_"+code)
today = dt.today()
strategy = Strategy()
strategy.set_data(indi, FS)
strategy.run()
buy = strategy.buy_signal
sell = strategy.sell_signal
unit = strategy.unit
remain_stock = buy['unit'].sum() - sell['unit'].sum()
remain = int(get_data(str(code)+".KS", today).iloc[0]['open'])*int(remain_stock)
total_buy = int((buy['open'].sum()))*unit
total_sell = int(sell['open'].sum())*unit
profit = int(remain) + int(total_sell) - int(total_buy)
if total_buy:
return_rate = profit / total_buy
trade = strategy.trade
total_return_per_day = trade['stock']+trade['cash']
residual = total_return_per_day - return_rate
sample_var = residual**2 / (trade.shape[0]-1)
sample_dev = np.sqrt(sample_var)
Rf=0.01
sharp = (return_rate - Rf) / (sample_dev)
results[code]['return'] = return_rate
results[code]['sharp'] = sharp
else:
print("No buy due to strict condition")
I have tried to make backtest code for investing into Korean stocks by using financial sheet and stock price sheet and indicator sheet.
And my code return error like the below.
UnboundLocalError Traceback (most recent call last)
<ipython-input-13-caf2b218f860> in <module>()
10 strategy = Strategy()
11 strategy.set_data(indi, FS)
---> 12 strategy.run()
13
14 buy = strategy.buy_signal
<ipython-input-12-2d41db386a22> in run(self)
84 curr_fs_cell = None
85
---> 86 if (curr_fs_cell == None) | (prev_fs_cell == None):
87 #print("fs data is empty")
88 continue
UnboundLocalError: local variable 'prev_fs_cell' referenced before assignment
Actually there is no global variable whose name is prev_fs_cell, but it is only in that class. Why this error occurs?
I use python to write the program to control the motor, and the error is shown as follows after running:"message": "No value for argument 'speed_a' in function call pylint(no-value-for-parameter)[93,21]' Below is the code I am using：
import serial
import time
#import threading
import struct
from binascii import unhexlify
from crcmod import mkCrcFun
import binascii
import crcmod
def check_code(byte0, byte1, speed_vel, speed_ang):
'计算校验码时需要输入的字节'
read = byte0+byte1+speed_vel+speed_ang #解析校验码要输入的前几位
read=(str(binascii.b2a_hex(read))[2:-1])
print (read)
return (read)
def crc16_modbus(read):
'输出的控制电机指令字节码'
crc16 =crcmod.mkCrcFun(0x18005,rev=True,initCrc=0xFFFF,xorOut=0x0000)
data = read.replace(" ","")
#print (data)
readcrcout=hex(crc16(unhexlify(data))).upper()
str_list = list(readcrcout)
if len(str_list) < 6:
str_list.insert(2, '0'*(6-len(str_list))) # 位数不足补0
crc_data = "".join(str_list)
#print(crc_data)
read = read.strip()+crc_data[4:]+crc_data[2:4]
read = read.encode('UTF-8')
return read
def motor_speed_vel(speed_v,speed_a):
'计算小车线速度speed_v'
DEC1 = speed_v
DEC2 = speed_a
byte2 =(struct.pack("i",DEC1)[-4:-3])#线速度
byte3 =(struct.pack("i",DEC1)[-3:-2])#线速度
speed_vel = byte2 + byte3
byte4 = (struct.pack("i",DEC2)[-4:-3]) #角速度两个字节
byte5 = (struct.pack("i",DEC2)[-3:-2])
speed_ang = byte4+byte5
print (speed_vel,speed_ang)
return (speed_vel,speed_ang)
motor_speed_vel(200,20)
'''
def motor_speed_ang(speed_a):
'角速度的speed_a的byte码'
DEC2 = speed_a
byte4 = (struct.pack("i",DEC2)[-4:-3]) #角速度两个字节
byte5 = (struct.pack("i",DEC2)[-3:-2])
speed_ang = byte4+byte5
print (speed_ang)
return (speed_ang)
motor_speed_ang(20)
#motor_speed_cal(200,20) #设定的 （线速度，角速度）
'''
motor_speed_mode = b'\x01\x2F\x60\x60\x00\x03\x00\x00\x00\x0D'
#motor_status = b'\x43\x50\x00\x51\x00\x68\x95'
motor_start = b'\x01\x44\x21\x00\x31\x00\x00\x01\x00\x01\x75\x34'
motor_stop = b'\x01\x44\x21\x00\x31\x00\x00\x00\x00\x00\xE5\x34' # AB轴失能
class SpeedMotor:
def __init__(self, device,speed_v,speed_a):
# 真实速度
self.rel_speed = 0
# 设置的速度
self.set_speed1 = speed_v
# 设置角速度
self.set_speed2 = speed_a
# 运行状态
self.run = False
# 故障状态
self.fault = None
# 电机电压
self.voltage = 0
# 电机电流
self.current = 0
# 设置串口通讯
self.serial = serial.Serial(device, 115200)
self.serial.timeout = 0
# 设置为速度模式
self.serial.write(motor_speed_mode)
time.sleep(0.1)
# 设置加减速度
# self.serial.write(b'\x0A\x14\x14\x32')
# time.sleep(0.1)
def motor_speed_set(self):
'速度设置'
byte0 = b'\x01'
byte1 = b'\xEA'
speed_vel = motor_speed_vel(self.set_speed1)
speed_ang = motor_speed_vel(self.set_speed2)
read = check_code(byte0, byte1, speed_vel, speed_ang)
speed_code = read
self.serial.write(speed_code)
def motor_start(self):
self.serial.write(motor_start)
self.run = True
def motor_stop(self):
self.run = False
self.serial.write(motor_stop)
m = SpeedMotor('COM5',200,20)
m.motor_start()
#for i in range(100):
# m.set_speed = i
time.sleep(15)
m.motor_stop()`
I don't know if it is because of the problem with my parameter call, or because of the structure of the code, and I am very uncertain about the correctness of the code structure, because I am a beginner.
My current GUI is to show target tracking from a TI processor to PC via UART. Sometimes, GUI crashes due to UART performance. But everytime I want to reset GUI, I also have to reset my TI device. So I want to modify python code so as to only re-open GUI, enter COM port without having to reset device whenever GUI crashes.
Currently, each time GUI crashes, even though I re-opened GUI but didn't reset device (power off & on), I can not continue tracking.
I put python code for GUI below.In this GUI python program, we have 2 blanks & button to enter UART COM port, 2 buttons to select & send configuration to device. Then it can begin tracking.
I'm considering to modify the part of sendCfg(self), but still not sure how to make it work as my purpose due to my lack of experience in Python & GUI design. Please help me with sample code for this task.
class Window(QDialog):
def __init__(self, parent=None, size=[]):
super(Window, self).__init__(parent)
#when running Gree Demo
self.Gree = 1
# set window toolbar options, and title. #deb_gp
self.setWindowFlags(
Qt.Window |
Qt.CustomizeWindowHint |
Qt.WindowTitleHint |
Qt.WindowMinimizeButtonHint |
Qt.WindowMaximizeButtonHint |
Qt.WindowCloseButtonHint
)
self.setWindowTitle("mmWave People Counting")
print('Python is ', struct.calcsize("P")*8, ' bit')
print('Python version: ', sys.version_info)
gridlay = QGridLayout()
gridlay.addWidget(self.comBox, 0,0,1,1)
gridlay.addWidget(self.statBox, 1,0,1,1)
gridlay.addWidget(self.configBox,2,0,1,1)
gridlay.addWidget(self.plotControlBox,3,0,1,1)
gridlay.addWidget(self.boxTab,4,0,1,1)
gridlay.addWidget(self.spBox,5,0,1,1)
gridlay.addWidget(self.graphTabs,0,1,6,1)
gridlay.addWidget(self.gw, 0, 2, 6, 1)
#gridlay.addWidget(self.demoData, 0,3,1,2)
#gridlay.addWidget(self.hPlot,1,3,4,2)
gridlay.setColumnStretch(0,1)
gridlay.setColumnStretch(1,3)
self.setLayout(gridlay)
#
# left side pane layout
#
def setConnectionLayout(self):
self.comBox = QGroupBox('Connect to Com Ports')
self.uartCom = QLineEdit('') #deb_gp
self.dataCom = QLineEdit('') #deb_gp
self.uartLabel = QLabel('UART COM:')
self.dataLabel = QLabel('DATA COM:')
self.connectStatus = QLabel('Not Connected')
self.connectButton = QPushButton('Connect')
self.connectButton.clicked.connect(self.connectCom)
self.configLabel = QLabel('Config Type:')
self.configType = QComboBox()
self.configType.addItems(["3D People Counting", "SDK Out of Box Demo", "Long Range People Detection", "Indoor False Detection Mitigation", "(Legacy) 2D People Counting", "(Legacy): Overhead People Counting"])
self.comLayout = QGridLayout()
self.comLayout.addWidget(self.uartLabel,0,0)
self.comLayout.addWidget(self.uartCom,0,1)
self.comLayout.addWidget(self.dataLabel,1,0)
self.comLayout.addWidget(self.dataCom,1,1)
self.comLayout.addWidget(self.configLabel,2,0)
self.comLayout.addWidget(self.configType,2,1)
self.comLayout.addWidget(self.connectButton,3,0)
self.comLayout.addWidget(self.connectStatus,3,1)
self.comBox.setLayout(self.comLayout)
def setConfigLayout(self):
self.configBox = QGroupBox('Configuration')
self.selectConfig = QPushButton('Select Configuration')
self.sendConfig = QPushButton('Send Configuration')
self.selectConfig.clicked.connect(self.selectCfg)
self.sendConfig.clicked.connect(self.sendCfg)
self.configTable = QTableWidget(5,2)
#set parameter names
self.configTable.setItem(0,0,QTableWidgetItem('Radar Parameter'))
self.configTable.setItem(0,1,QTableWidgetItem('Value'))
self.configTable.setItem(1,0,QTableWidgetItem('Max Range'))
self.configTable.setItem(2,0,QTableWidgetItem('Range Resolution'))
self.configTable.setItem(3,0,QTableWidgetItem('Max Velocity'))
self.configTable.setItem(4,0,QTableWidgetItem('Velcoity Resolution'))
self.configLayout = QVBoxLayout()
self.configLayout.addWidget(self.selectConfig)
self.configLayout.addWidget(self.sendConfig)
self.configLayout.addWidget(self.configTable)
#self.configLayout.addStretch(1)
self.configBox.setLayout(self.configLayout)
def updateGraph(self, parsedData):
updateStart = int(round(time.time()*1000))
self.useFilter = 0
classifierOutput = []
pointCloud = parsedData[0]
targets = parsedData[1]
indexes = parsedData[2]
numPoints = parsedData[3]
numTargets = parsedData[4]
self.frameNum = parsedData[5]
fail = parsedData[6]
classifierOutput = parsedData[7]
fallDetEn = 0
indicesIn = []
if (fail != 1):
#left side
pointstr = 'Points: '+str(numPoints)
targetstr = 'Targets: '+str(numTargets)
self.numPointsDisplay.setText(pointstr)
self.numTargetsDisplay.setText(targetstr)
#right side fall detection
peopleStr = 'Number of Detected People: '+str(numTargets)
if (numTargets == 0):
fdestr = 'Fall Detection Disabled - No People Detected'
elif (numTargets == 1):
fdestr = 'Fall Detection Enabled'
fallDetEn = 1
elif (numTargets > 1):
fdestr = 'Fall Detected Disabled - Too Many People'
#self.numDetPeople.setText(peopleStr)
#self.fallDetEnabled.setText(fdestr)
if (len(targets) < 13):
targets = []
classifierOutput = []
if (fail):
return
#remove static points
if (self.configType.currentText() == '3D People Counting'):
if (not self.staticclutter.isChecked()):
statics = np.where(pointCloud[3,:] == 0)
try:
firstZ = statics[0][0]
numPoints = firstZ
pointCloud = pointCloud[:,:firstZ]
indexes = indexes[:,:self.previousFirstZ]
self.previousFirstZ = firstZ
except:
firstZ = -1
#point cloud persistence
fNum = self.frameNum%10
if (numPoints):
self.previousCloud[:5,:numPoints,fNum] = pointCloud[:5,:numPoints]
self.previousCloud[5,:len(indexes),fNum] = indexes
self.previousPointCount[fNum]=numPoints
#plotting 3D - get correct point cloud (persistent points and synchronize the frame)
if (self.configType.currentText() == 'SDK3xPeopleCount'):
pointIn = pointCloud
else:
totalPoints = 0
persistentFrames = int(self.persistentFramesInput.currentText())
#allocate new array for all the points
for i in range(1,persistentFrames+1):
totalPoints += self.previousPointCount[fNum-i]
pointIn = np.zeros((5,int(totalPoints)))
indicesIn = np.ones((1, int(totalPoints)))*255
totalPoints = 0
#fill array
for i in range(1,persistentFrames+1):
prevCount = int(self.previousPointCount[fNum-i])
pointIn[:,totalPoints:totalPoints+prevCount] = self.previousCloud[:5,:prevCount,fNum-i]
if (numTargets > 0):
indicesIn[0,totalPoints:totalPoints+prevCount] = self.previousCloud[5,:prevCount,fNum-i]
totalPoints+=prevCount
if (self.graphFin):
self.plotstart = int(round(time.time()*1000))
self.graphFin = 0
if (self.threeD):
try:
indicesIn = indicesIn[0,:]
except:
indicesIn = []
self.get_thread = updateQTTargetThread3D(pointIn, targets, indicesIn, self.scatter, self.pcplot, numTargets, self.ellipsoids, self.coordStr, classifierOutput, self.zRange, self.gw, self.plotByIndex.isChecked(), self.plotTracks.isChecked(), self.bbox,self.boundaryBoxes[0]['checkEnable'].isChecked())
self.get_thread.done.connect(self.graphDone)
self.get_thread.start(priority=QThread.HighestPriority-1)
else:
npc = pointIn[0:2,:]
print (np.shape(npc))
self.legacyThread = update2DQTGraphThread(npc, targets, numTargets, indexes, numPoints, self.trailData, self.activeTrail, self.trails, self.scatter2D, self.gatingScatter)
self.legacyThread.done.connect(self.graphDone)
self.legacyThread.start(priority=QThread.HighestPriority-1)
else:
return
#pointIn = self.previousCloud[:,:int(self.previousPointCount[fNum-1]),fNum-1]
#state tracking
if (numTargets > 0):
self.lastFrameHadTargets = True
else:
self.lastFrameHadTargets = False
if (numTargets):
self.lastTID = targets[0,:]
else:
self.lastTID = []
def graphDone(self):
plotend = int(round(time.time()*1000))
plotime = plotend - self.plotstart
try:
if (self.frameNum > 1):
self.averagePlot = (plotime*1/self.frameNum) + (self.averagePlot*(self.frameNum-1)/(self.frameNum))
else:
self.averagePlot = plotime
except:
self.averagePlot = plotime
self.graphFin = 1
pltstr = 'Average Plot time: '+str(plotime)[:5] + ' ms'
fnstr = 'Frame: '+str(self.frameNum)
self.frameNumDisplay.setText(fnstr)
self.plotTimeDisplay.setText(pltstr)
def resetFallText(self):
self.fallAlert.setText('Standing')
self.fallPic.setPixmap(self.standingPicture)
self.fallResetTimerOn = 0
def updateFallThresh(self):
try:
newThresh = float(self.fallThreshInput.text())
self.fallThresh = newThresh
self.fallThreshMarker.setPos(self.fallThresh)
except:
print('No numberical threshold')
def connectCom(self):
#get parser
self.parser = uartParserSDK(type=self.configType.currentText())
self.parser.frameTime = self.frameTime
print('Parser type: ',self.configType.currentText())
#init threads and timers
self.uart_thread = parseUartThread(self.parser)
if (self.configType.currentText() != 'Replay'):
self.uart_thread.fin.connect(self.parseData)
self.uart_thread.fin.connect(self.updateGraph)
self.parseTimer = QTimer()
self.parseTimer.setSingleShot(False)
self.parseTimer.timeout.connect(self.parseData)
try:
uart = "COM"+ self.uartCom.text() #deb_gp
data = "COM"+ self.dataCom.text() #deb_gp
#TODO: find the serial ports automatically.
self.parser.connectComPorts(uart, data)
self.connectStatus.setText('Connected') #deb_gp
self.connectButton.setText('Disconnect') #deb_gp
#TODO: create the disconnect button action
except:
self.connectStatus.setText('Unable to Connect')
if (self.configType.currentText() == "Replay"):
self.connectStatus.setText('Replay')
if (self.configType.currentText() == "Long Range People Detection"):
self.frameTime = 400
#
# Select and parse the configuration file
# TODO select the cfgfile automatically based on the profile.
def selectCfg(self):
try:
self.parseCfg(self.selectFile())
except:
print('No cfg file selected!')
def selectFile(self):
fd = QFileDialog()
filt = "cfg(*.cfg)"
filename = fd.getOpenFileName(directory='./../chirp_configs',filter=filt) #deb_gp - added folder name
return filename[0]
def parseCfg(self, fname):
cfg_file = open(fname, 'r')
self.cfg = cfg_file.readlines()
counter = 0
chirpCount = 0
for line in self.cfg:
args = line.split()
if (len(args) > 0):
if (args[0] == 'cfarCfg'):
zy = 4
#self.cfarConfig = {args[10], args[11], '1'}
elif (args[0] == 'AllocationParam'):
zy=3
#self.allocConfig = tuple(args[1:6])
elif (args[0] == 'GatingParam'):
zy=2
#self.gatingConfig = tuple(args[1:4])
elif (args[0] == 'SceneryParam' or args[0] == 'boundaryBox'):
self.boundaryLine = counter
self.profile['leftX'] = float(args[1])
self.profile['rightX'] = float(args[2])
self.profile['nearY'] = float(args[3])
self.profile['farY'] = float(args[4])
self.setBoundaryTextVals(self.profile)
self.boundaryBoxes[0]['checkEnable'].setChecked(True)
elif (args[0] == 'staticBoundaryBox'):
self.staticLine = counter
elif (args[0] == 'profileCfg'):
self.profile['startFreq'] = float(args[2])
self.profile['idle'] = float(args[3])
self.profile['adcStart'] = float(args[4])
self.profile['rampEnd'] = float(args[5])
self.profile['slope'] = float(args[8])
self.profile['samples'] = float(args[10])
self.profile['sampleRate'] = float(args[11])
print(self.profile)
elif (args[0] == 'frameCfg'):
self.profile['numLoops'] = float(args[3])
self.profile['numTx'] = float(args[2])+1
elif (args[0] == 'chirpCfg'):
chirpCount += 1
elif (args[0] == 'sensorPosition'):
self.profile['sensorHeight'] = float(args[1])
self.profile['az_tilt'] = float(args[2])
self.profile['elev_tilt'] = float(args[3])
counter += 1
maxRange = self.profile['sampleRate']*1e3*0.9*3e8/(2*self.profile['slope']*1e12)
#update boundary box
self.drawBoundaryGrid(maxRange)
#update chirp table values
bw = self.profile['samples']/(self.profile['sampleRate']*1e3)*self.profile['slope']*1e12
rangeRes = 3e8/(2*bw)
Tc = (self.profile['idle']*1e-6 + self.profile['rampEnd']*1e-6)*chirpCount
lda = 3e8/(self.profile['startFreq']*1e9)
maxVelocity = lda/(4*Tc)
velocityRes = lda/(2*Tc*self.profile['numLoops']*self.profile['numTx'])
self.configTable.setItem(1,1,QTableWidgetItem(str(maxRange)[:5]))
self.configTable.setItem(2,1,QTableWidgetItem(str(rangeRes)[:5]))
self.configTable.setItem(3,1,QTableWidgetItem(str(maxVelocity)[:5]))
self.configTable.setItem(4,1,QTableWidgetItem(str(velocityRes)[:5]))
#update sensor position
self.az_tilt.setText(str(self.profile['az_tilt']))
self.elev_tilt.setText(str(self.profile['elev_tilt']))
self.s_height.setText(str(self.profile['sensorHeight']))
def sendCfg(self):
try:
if (self.configType.currentText() != "Replay"):
self.parser.sendCfg(self.cfg)
self.configSent = 1
self.parseTimer.start(self.frameTime)
except:
print ('No cfg file selected!')
# Needed ?? deb_gp
# def setParser(self, uParser):
# self.parser = uParser
def parseData(self):
self.uart_thread.start(priority=QThread.HighestPriority)
def whoVisible(self):
if (self.threeD):
self.threeD = 0
else:
self.threeD = 1
print('3d: ', self.threeD)
if __name__ == '__main__':
if (compileGui):
appctxt = ApplicationContext()
app = QApplication(sys.argv)
screen = app.primaryScreen()
size = screen.size()
main = Window(size=size)
main.show()
exit_code = appctxt.app.exec_()
sys.exit(exit_code)
else:
app = QApplication(sys.argv)
screen = app.primaryScreen()
size = screen.size()
main = Window(size=size)
main.show()
sys.exit(app.exec_())