Develop Reference

parallelized tasks are not ditributed a cross available cpus

As shown in the code posted below in section DecoupleGridCellsProfilerLoopsPool, the run() is called as much times as the contents of the self.__listOfLoopDecouplers and it works as it supposed to be, i mean the parallelization is working duly.
as shown in the same section, DecoupleGridCellsProfilerLoopsPool.pool.map returns results and i populate some lists,lets discuss the list names self.__iterablesOfZeroCoverageCell it contains number of objects of type gridCellInnerLoopsIteratorsForZeroCoverageModel.
After that, i created the pool ZeroCoverageCellsProcessingPool with the code as posted below as well.
The problem i am facing, is the parallelized code in ZeroCoverageCellsProcessingPool is very slow and the visulisation of the cpu tasks shows that there are no processes work in parallel as shown in the video contained in url posted below.
i was suspicious about the pickling issues related to when parallelizing the code in ZeroCoverageCellsProcessingPool,so i removed the enitre body of the run() in ZeroCoverageCellsProcessingPool. however, it shows no change in the behaviour of the parallelized code.
also the url posted below shown how the parallelized methoth of ZeroCoverageCellsProcessingPool behaves.
given the code posted below, please let me know why the parallelization does not work for code in ZeroCoverageCellsProcessingPool
output url:please click the link
output url
DecoupleGridCellsProfilerLoopsPool
def postTask(self):
self.__postTaskStartTime = time.time()
with Pool(processes=int(config['MULTIPROCESSING']['proceses_count'])) as DecoupleGridCellsProfilerLoopsPool.pool:
self.__chunkSize = PoolUtils.getChunkSize(lst=self.__listOfLoopDecouplers,cpuCount=int(config['MULTIPROCESSING']['cpu_count']))
logger.info(f"DecoupleGridCellsProfilerLoopsPool.self.__chunkSize(task per processor):{self.__chunkSize}")
for res in DecoupleGridCellsProfilerLoopsPool.pool.map(self.run,self.__listOfLoopDecouplers,chunksize=self.__chunkSize):
if res[0] is not None and res[1] is None and res[2] is None:
self.__iterablesOfNoneZeroCoverageCell.append(res[0])
elif res[1] is not None and res[0] is None and res[2] is None:
self.__iterablesOfZeroCoverageCell.append(res[1])
elif res[2] is not None and res[0] is None and res[1] is None:
self.__iterablesOfNoDataCells.append(res[2])
else:
raise Exception (f"WTF.")
DecoupleGridCellsProfilerLoopsPool.pool.join()
assert len(self.__iterablesOfNoneZeroCoverageCell)+len(self.__iterablesOfZeroCoverageCell)+len(self.__iterablesOfNoDataCells) == len(self.__listOfLoopDecouplers)
zeroCoverageCellsProcessingPool = ZeroCoverageCellsProcessingPool(self.__devModeForWSAWANTIVer2,self.__iterablesOfZeroCoverageCell)
zeroCoverageCellsProcessingPool.postTask()
def run(self,param:LoopDecoupler):
row = param.getRowValue()
col = param.getColValue()
elevationsTIFFWindowedSegmentContents = param.getElevationsTIFFWindowedSegment()
verticalStep = param.getVericalStep()
horizontalStep = param.getHorizontalStep()
mainTIFFImageDatasetContents = param.getMainTIFFImageDatasetContents()
NDVIsTIFFWindowedSegmentContentsInEPSG25832 = param.getNDVIsTIFFWindowedSegmentContentsInEPSG25832()
URLOrFilePathForElevationsTIFFDatasetInEPSG25832 = param.getURLOrFilePathForElevationsTIFFDatasetInEPSG25832()
threshold = param.getThreshold()
rowsCnt = 0
colsCnt = 0
pixelsValuesSatisfyThresholdInTIFFImageDatasetCnt = 0
pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt = int(config['window']['width']) * int(config['window']['height'])
pixelsWithNoDataValueInTIFFImageDatasetCnt = int(config['window']['width']) * int(config['window']['height'])
_pixelsValuesSatisfyThresholdInNoneZeroCoverageCell = []
_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell = []
_pixelsValuesInNoDataCell = []
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel = None
gridCellInnerLoopsIteratorsForZeroCoverageModel = None
gridCellInnerLoopsIteratorsForNoDataCellsModel = None
for x in range(row,row + verticalStep):
if rowsCnt == verticalStep:
rowsCnt = 0
for y in range(col,col + horizontalStep):
if colsCnt == horizontalStep:
colsCnt = 0
pixelValue = mainTIFFImageDatasetContents[0][x][y]
# windowIOUtils.writeContentsToFile(windowIOUtils.getPathToOutputDir()+"/"+config['window']['file_name']+".{0}".format(config['window']['file_extension']), "pixelValue:{0}\n".format(pixelValue))
if pixelValue >= float(threshold):
pixelsValuesSatisfyThresholdInTIFFImageDatasetCnt+=1
_pixelsValuesSatisfyThresholdInNoneZeroCoverageCell.append(elevationsTIFFWindowedSegmentContents[0][rowsCnt][colsCnt])
elif ((pixelValue < float(threshold)) and (pixelValue > float(config['TIFF']['no_data_value']))):
pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt-=1
_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell.append(elevationsTIFFWindowedSegmentContents[0][rowsCnt][colsCnt])
elif (pixelValue <= float(config['TIFF']['no_data_value'])):
pixelsWithNoDataValueInTIFFImageDatasetCnt-=1
_pixelsValuesInNoDataCell.append(elevationsTIFFWindowedSegmentContents[0][rowsCnt][colsCnt])
else:
raise Exception ("WTF.Exception: unhandled condition for pixel value: {0}".format(pixelValue))
# _pixelCoordinatesInWindow.append([x,y])
colsCnt+=1
rowsCnt+=1
'''Grid-cell classfication'''
if (pixelsValuesSatisfyThresholdInTIFFImageDatasetCnt > 0):
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel = GridCellInnerLoopsIteratorsForNoneZeroCoverageModel()
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setRowValue(row)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setColValue(col)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setVericalStep(verticalStep)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setHorizontalStep(horizontalStep)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setMainTIFFImageDatasetContents(mainTIFFImageDatasetContents)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setNDVIsTIFFWindowedSegmentContentsInEPSG25832(NDVIsTIFFWindowedSegmentContentsInEPSG25832)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setURLOrFilePathForElevationsTIFFDatasetInEPSG25832(URLOrFilePathForElevationsTIFFDatasetInEPSG25832)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setPixelsValuesSatisfyThresholdInTIFFImageDatasetCnt(pixelsValuesSatisfyThresholdInTIFFImageDatasetCnt)
gridCellInnerLoopsIteratorsForNoneZeroCoverageModel.setPixelsValuesSatisfyThresholdInNoneZeroCoverageCell(_pixelsValuesSatisfyThresholdInNoneZeroCoverageCell)
elif (pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt < (int(config['window']['width']) * int(config['window']['height'])) and pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt >= 0):
gridCellInnerLoopsIteratorsForZeroCoverageModel = GridCellInnerLoopsIteratorsForZeroCoverageModel()
gridCellInnerLoopsIteratorsForZeroCoverageModel.setRowValue(row)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setColValue(col)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setVericalStep(verticalStep)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setHorizontalStep(horizontalStep)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setMainTIFFImageDatasetContents(mainTIFFImageDatasetContents)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setNDVIsTIFFWindowedSegmentContentsInEPSG25832(NDVIsTIFFWindowedSegmentContentsInEPSG25832)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setURLOrFilePathForElevationsTIFFDatasetInEPSG25832(URLOrFilePathForElevationsTIFFDatasetInEPSG25832)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt(pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setPixelsWithNoDataValueInTIFFImageDatasetCnt(pixelsWithNoDataValueInTIFFImageDatasetCnt)
gridCellInnerLoopsIteratorsForZeroCoverageModel.setPixelsValuesDoNotSatisfyThresholdInZeroCoverageCell(_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell)
elif (pixelsWithNoDataValueInTIFFImageDatasetCnt == 0):
gridCellInnerLoopsIteratorsForNoDataCellsModel = GridCellInnerLoopsIteratorsForNoDataCellsModel()
gridCellInnerLoopsIteratorsForNoDataCellsModel.setRowValue(row)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setColValue(col)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setVericalStep(verticalStep)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setHorizontalStep(horizontalStep)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setMainTIFFImageDatasetContents(mainTIFFImageDatasetContents)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setNDVIsTIFFWindowedSegmentContentsInEPSG25832(NDVIsTIFFWindowedSegmentContentsInEPSG25832)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setURLOrFilePathForElevationsTIFFDatasetInEPSG25832(URLOrFilePathForElevationsTIFFDatasetInEPSG25832)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setPixelsWithNoDataValueInTIFFImageDatasetCnt(pixelsWithNoDataValueInTIFFImageDatasetCnt)
gridCellInnerLoopsIteratorsForNoDataCellsModel.setPixelsValuesInNoDataCell(_pixelsValuesInNoDataCell)
if gridCellInnerLoopsIteratorsForZeroCoverageModel is not None:
gridCellInnerLoopsIteratorsForZeroCoverageModel.setPixelsWithNoDataValueInTIFFImageDatasetCnt(pixelsWithNoDataValueInTIFFImageDatasetCnt)
else:
raise Exception (f"WTF.")
return gridCellInnerLoopsIteratorsForNoneZeroCoverageModel,gridCellInnerLoopsIteratorsForZeroCoverageModel,gridCellInnerLoopsIteratorsForNoDataCellsModel
ZeroCoverageCellsProcessingPool:
def postTask(self):
self.__postTaskStartTime = time.time()
"""to collect results per each row
"""
resAllCellsForGridCellsClassifications = []
# NDVIs
resAllCellsForNDVITIFFDetailsForZeroCoverageCell = []
# area of coverage
resAllCellsForAreaOfCoverageForZeroCoverageCell = []
# interception
resAllCellsForInterceptionForZeroCoverageCell = []
# fourCornersOfWindowInEPSG25832
resAllCellsForFourCornersOfWindowInEPSG25832ZeroCoverageCell = []
# outFromEPSG25832ToEPSG4326-lists
resAllCellsForOutFromEPSG25832ToEPSG4326ForZeroCoverageCells = []
# fourCornersOfWindowsAsGeoJSON
resAllCellsForFourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell = []
# calculatedCenterPointInEPSG25832
resAllCellsForCalculatedCenterPointInEPSG25832ForZeroCoverageCell = []
# centerPointsOfWindowInImageCoordinatesSystem
resAllCellsForCenterPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell = []
# pixelValuesOfCenterPoints
resAllCellsForPixelValuesOfCenterPointsForZeroCoverageCell = []
# centerPointOfKeyWindowAsGeoJSONInEPSG4326
resAllCellsForCenterPointOfKeyWindowAsGeoJSONInEPSG4326ForZeroCoverageCell = []
# centerPointInEPSG4326
resAllCellsForCenterPointInEPSG4326ForZeroCoveringCell = []
# average heights
resAllCellsForAverageHeightsForZeroCoverageCell = []
# pixels values
resAllCellsForPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCell = []
# area Of Coverage
resAllCellsForAreaOfCoverageForZeroCoverageCell = []
resAllCellsForPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt = []
noneKeyWindowCnt=0
# center points as string
centerPointsAsStringForZeroCoverageCell = ""
with Pool(processes=int(config['MULTIPROCESSING']['proceses_count'])) as ZeroCoverageCellsProcessingPool.pool:
self.__chunkSize = PoolUtils.getChunkSize(lst=self.__iterables,cpuCount=int(config['MULTIPROCESSING']['cpu_count']))
logger.info(f"ZeroCoverageCellsProcessingPool.self.__chunkSize(task per processor):{self.__chunkSize}")
for res in ZeroCoverageCellsProcessingPool.pool.map(func=self.run,iterable=self.__iterables,chunksize=self.__chunkSize):
resAllCellsForGridCellsClassifications.append(res[0])
# NDVIs
resAllCellsForNDVITIFFDetailsForZeroCoverageCell.append(res[1])
# area of coverage
resAllCellsForAreaOfCoverageForZeroCoverageCell.append(res[2])
# interception
resAllCellsForInterceptionForZeroCoverageCell.append(res[3])
# fourCornersOfWindowInEPSG25832
resAllCellsForFourCornersOfWindowInEPSG25832ZeroCoverageCell.append(res[4])
# outFromEPSG25832ToEPSG4326-lists
resAllCellsForOutFromEPSG25832ToEPSG4326ForZeroCoverageCells.append(res[5])
# fourCornersOfWindowsAsGeoJSONInEPSG4326
resAllCellsForFourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell.append(res[6])
# calculatedCenterPointInEPSG25832
resAllCellsForCalculatedCenterPointInEPSG25832ForZeroCoverageCell.append(res[7])
# centerPointsOfWindowInImageCoordinatesSystem
resAllCellsForCenterPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell.append(res[8])
# pixelValuesOfCenterPoints
resAllCellsForPixelValuesOfCenterPointsForZeroCoverageCell.append(res[9])
# centerPointInEPSG4326
resAllCellsForCenterPointInEPSG4326ForZeroCoveringCell.append(res[10])
# centerPointOfKeyWindowAsGeoJSONInEPSG4326
resAllCellsForCenterPointOfKeyWindowAsGeoJSONInEPSG4326ForZeroCoverageCell.append(res[11])
# average heights
resAllCellsForAverageHeightsForZeroCoverageCell.append(res[12])
# pixels values
resAllCellsForPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCell.append(res[13])
# pixelsValues cnt
resAllCellsForPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt.append(res[14])
noneKeyWindowCnt +=res[15]
# centerPoints-As-String
if (res[16] is not None):
centerPointsAsStringForZeroCoverageCell+=str(res[16])
assert noneKeyWindowCnt == len(self.__iterables)
ZeroCoverageCellsProcessingPool.pool.close()
ZeroCoverageCellsProcessingPool.pool.terminate()
ZeroCoverageCellsProcessingPool.pool.join()
return
def run(self,params:GridCellInnerLoopsIteratorsForZeroCoverageModel):
if params is not None:
logger.info(f"Processing zero coverage cell #(row{params.getRowValue()},col:{params.getColValue()})")
row = params.getRowValue()
col = params.getColValue()
mainTIFFImageDatasetContents = params.getMainTIFFImageDatasetContents()
NDVIsTIFFWindowedSegmentContentsInEPSG25832 = params.getNDVIsTIFFWindowedSegmentContentsInEPSG25832()
URLOrFilePathForElevationsTIFFDatasetInEPSG25832 = params.getURLOrFilePathForElevationsTIFFDatasetInEPSG25832()
datasetElevationsTIFFInEPSG25832 = rasterio.open(URLOrFilePathForElevationsTIFFDatasetInEPSG25832,'r')
_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell = params.getPixelsValuesDoNotSatisfyThresholdInZeroCoverageCell()
pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt = params.getPixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt()
countOfNoDataCells = params.getPixelsWithNoDataValueInTIFFImageDatasetCnt()
outFromEPSG25832ToEPSG4326ForZeroCoverageCells = []
fourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell = []
ndviTIFFDetailsForZeroCoverageCell = NDVITIFFDetails(None,None,None).getNDVIValuePer10mX10m()
"""area of coverage per grid-cell"""
areaOfCoverageForZeroCoverageCell = None
""""interception"""
interceptionForZeroCoverageCell = None
CntOfNDVIsWithNanValueInZeroCoverageCell = 0
fourCornersOfWindowInEPSG25832ZeroCoverageCell = None
outFromEPSG25832ToEPSG4326ForZeroCoverageCells = []
fourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell = []
calculatedCenterPointInEPSG25832ForZeroCoverageCell = None
centerPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell = None
pixelValuesOfCenterPointsOfZeroCoverageCell = None
centerPointInEPSG4326ForZeroCoveringCell = None
centerPointOfKeyWindowAsGeoJSONInEPSG4326ForZeroCoverageCell = None
centerPointsAsStringForZeroCoverageCell = None
"""average heights"""
averageHeightsForZeroCoverageCell = None
gridCellClassifiedAs = GridCellClassifier.ZERO_COVERAGE_CELL.value
cntOfNoneKeyWindow = 1
ndviTIFFDetailsForZeroCoverageCell = NDVITIFFDetails(ulX=row//int(config['ndvi']['resolution_height']),ulY=col//int(config['ndvi']['resolution_width']),dataset=NDVIsTIFFWindowedSegmentContentsInEPSG25832).getNDVIValuePer10mX10m()
"""area of coverage per grid-cell"""
areaOfCoverageForZeroCoverageCell = round(AreaOfCoverageDetails(pixelsCount=(int(config['window']['width']) * int(config['window']['height'])) - (pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt + countOfNoDataCells)).getPercentageOfAreaOfCoverage(),2)
""""interception"""
if math.isnan(ndviTIFFDetailsForZeroCoverageCell):
# ndviTIFFDetailsForZeroCoverageCell = 0
CntOfNDVIsWithNanValueInZeroCoverageCell = 1
interceptionForZeroCoverageCell = config['sentinel_values']['interception']
else:
Indvi = INDVI()
Ic = Indvi.calcInterception(ndviTIFFDetailsForZeroCoverageCell)
Pc=areaOfCoverageForZeroCoverageCell,"""percentage of coverage"""
Pnc=float((int(config['window']['width'])*int(config['window']['height'])) - areaOfCoverageForZeroCoverageCell),"""percentage of non-coverage"""
Inc=float(config['interception']['noneCoverage']),"""interception of none-coverage"""
I=(float(Pc[0])*(Ic))+float((Pnc[0]*Inc[0]))
interceptionForZeroCoverageCell = round(I,2)
if I != 10 and I != float('nan'):
logger.error(f"ndviTIFFDetailsForZeroCoverageCell:{ndviTIFFDetailsForZeroCoverageCell}")
logger.error(f"I:{I}")
fourCornersOfWindowInEPSG25832ZeroCoverageCell = RasterIOPackageUtils.convertFourCornersOfWindowFromImageCoordinatesToCRSByCoordinatesOfCentersOfPixelsMethodFor(row,col,int(config['window']['height']),int(config['window']['width']),datasetElevationsTIFFInEPSG25832)
for i in range(0,len(fourCornersOfWindowInEPSG25832ZeroCoverageCell)):
# fourCornersOfKeyWindowInEPSG4326.append(RasterIOPackageUtils.convertCoordsToDestEPSGForDataset(fourCornersOfWindowInEPSG25832[i],datasetElevationsTIFFInEPSG25832,destEPSG=4326))
outFromEPSG25832ToEPSG4326ForZeroCoverageCells.append(OSGEOUtils.fromEPSG25832ToEPSG4326(fourCornersOfWindowInEPSG25832ZeroCoverageCell[i])) # resultant coords order is in form of lat,lon and it must be in lon,lat.thus, out[1]-lat out[0]-lon
"""fourCornersOfWindowsAsGeoJSONInEPSG4326"""
fourCornersOfWindowInEPSG4326 = []
for i in range(0,len(outFromEPSG25832ToEPSG4326ForZeroCoverageCells)):
fourCornersOfWindowInEPSG4326.append(([outFromEPSG25832ToEPSG4326ForZeroCoverageCells[i][1]],[outFromEPSG25832ToEPSG4326ForZeroCoverageCells[i][0]]))
fourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell.append(jsonUtils.buildFeatureCollectionAsGeoJSONForFourCornersOfKeyWindow(fourCornersOfWindowInEPSG4326[0],fourCornersOfWindowInEPSG4326[1],fourCornersOfWindowInEPSG4326[2],fourCornersOfWindowInEPSG4326[3],areaOfCoverageForZeroCoverageCell))
# debugIOUtils.writeContentsToFile(debugIOUtils.getPathToOutputDir()+"/"+"NDVIsPer10mX10mForKeyWindow"+config['window']['file_name']+".{0}".format(config['window']['file_extension']),"{0}\n".format(NDVIsPer10mX10mForKeyWindow))
"""
building geojson object for a point "center-point" to visualize it.
"""
calculatedCenterPointInEPSG25832ForZeroCoverageCell = MiscUtils.calculateCenterPointsGivenLLOfGridCell(fourCornersOfWindowInEPSG25832ZeroCoverageCell[1])#lower-left corner
centerPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell = RasterIOPackageUtils.convertFromCRSToImageCoordinatesSystemFor(calculatedCenterPointInEPSG25832ForZeroCoverageCell[0],calculatedCenterPointInEPSG25832ForZeroCoverageCell[1],datasetElevationsTIFFInEPSG25832)
pixelValuesOfCenterPointsOfZeroCoverageCell = mainTIFFImageDatasetContents[0][centerPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell[0]][centerPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell[1]]
centerPointInEPSG4326ForZeroCoveringCell = RasterIOPackageUtils.convertCoordsToDestEPSGForDataset(calculatedCenterPointInEPSG25832ForZeroCoverageCell,datasetElevationsTIFFInEPSG25832,destEPSG=4326)
centerPointOfKeyWindowAsGeoJSONInEPSG4326ForZeroCoverageCell = jsonUtils.buildGeoJSONForPointFor(centerPointInEPSG4326ForZeroCoveringCell)
"""average heights"""
averageHeightsForZeroCoverageCell = round(MiscUtils.getAverageFor(_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell),2)
assert len(_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell) > 0 and (len(_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell) <= (int(config['window']['width']) * int(config['window']['height'])) )
"""the following code block is for assertion only"""
if self.__devModeForWSAWANTIVer2 == config['DEVELOPMENT_MODE']['debug']:
assert pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt >= 0 and (pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt < (int(config['window']['width']) * int(config['window']['height'])) )
assert (pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt+countOfNoDataCells) == (int(config['window']['width']) * int(config['window']['height']))
print(f"profiling for gridCellClassifiedAs:{gridCellClassifiedAs}....>pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt:{pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt}")
print(f"profiling for gridCellClassifiedAs:{gridCellClassifiedAs}....>countOfNoDataCells:{countOfNoDataCells}")
pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt = (int(config['window']['width']) * int(config['window']['height'])) - (pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt + countOfNoDataCells)
assert pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt == 0, (f"WTF.")
print(f"profiling for gridCellClassifiedAs:{gridCellClassifiedAs}....>computed pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt:{pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt}")
print(f"\n")
centerPointAsTextInWKTInEPSG3857 = CoordinatesUtils.buildWKTPointFormatForSinglePointFor(calculatedCenterPointInEPSG25832ForZeroCoverageCell[0],calculatedCenterPointInEPSG25832ForZeroCoverageCell[1])
s = centerPointAsTextInWKTInEPSG3857.replace("POINT","")
s = s.replace("(","")
s = s.replace(")","")
s = s.strip()
s = s.split(" ")
centerPointsAsStringForZeroCoverageCell = s[0] + "\t" + s[1] + "\n"
centerPointsAsStringForZeroCoverageCell = centerPointsAsStringForZeroCoverageCell.replace('\'',"")
return gridCellClassifiedAs,ndviTIFFDetailsForZeroCoverageCell,areaOfCoverageForZeroCoverageCell,interceptionForZeroCoverageCell,fourCornersOfWindowInEPSG25832ZeroCoverageCell,outFromEPSG25832ToEPSG4326ForZeroCoverageCells,fourCornersOfWindowsAsGeoJSONInEPSG4326ForZeroCoverageCell,calculatedCenterPointInEPSG25832ForZeroCoverageCell,centerPointsOfWindowInImageCoordinatesSystemForZeroCoverageCell,pixelValuesOfCenterPointsOfZeroCoverageCell,centerPointInEPSG4326ForZeroCoveringCell,centerPointOfKeyWindowAsGeoJSONInEPSG4326ForZeroCoverageCell,averageHeightsForZeroCoverageCell,np.array(_pixelsValuesDoNotSatisfyThresholdInZeroCoverageCell).tolist(),pixelsValuesDoNotSatisfyThresholdInTIFFImageDatasetCnt,cntOfNoneKeyWindow,centerPointsAsStringForZeroCoverageCell,CntOfNDVIsWithNanValueInZeroCoverageCell

ValueError: substring not found on lip reading code

This is what I have gotten while trying to run step 3 of this source code:
https://github.com/carykh/lazykh
Error:
Traceback (most recent call last):
File "C:\Users\User\Desktop\lazykh-main\code\scheduler.py", line 93, in
OS_nextIndex = originalScript.index(wordString,OS_IndexAt)+len(wordString)
ValueError: substring not found
Code:
import argparse
import os.path
import json
import numpy as np
import random
def addPhoneme(p, t):
global prevPhoneme
global f
if p != prevPhoneme:
strings[4] += (str.format('{0:.3f}', t)+",phoneme,"+p+"\n")
prevPhoneme = p
def pickNewPose(t):
global pose
global prevPose
global POSE_COUNT
global prevPhoneme
global f
newPose = -1
while newPose == -1 or newPose == pose or newPose == prevPose:
newPose = int(random.random()*POSE_COUNT)
prevPose = pose
pose = newPose
strings[3] += (str.format('{0:.3f}', t)+",pose,"+str(pose)+"\n")
prevPhoneme = "na"
strings = [""]*5
POSE_COUNT = 5
emotions = {}
emotions["explain"] = 0
emotions["happy"] = 1
emotions["sad"] = 2
emotions["angry"] = 3
emotions["confused"] = 4
emotions["rq"] = 5
mouthList = [["aa","a"],["ae","a"],["ah","a"],["ao","a"],["aw","au"],
["ay","ay"],["b","m"],["ch","t"],["d","t"],["dh","t"],
["eh","a"],["er","u"],["ey","ay"],["f","f"],["g","t"],
["hh","y"],["ih","a"],["iy","ay"],["jh","t"],["k","t"],
["l","y"],["m","m"],["n","t"],["ng","t"],["ow","au"],
["oy","ua"],["p","m"],["r","u"],["s","t"],["sh","t"],
["t","t"],["th","t"],["uh","u"],["uw","u"],["v","f"],
["w","u"],["y","y"],["z","t"],["zh","t"],
["oov","m"]] # For unknown phonemes, the stick figure will just have a closed mouth ("mmm")
mouths = {}
for x in mouthList:
mouths[x[0]] = x[1]
ENDING_PHONEME = "m"
STOPPERS = [",",";",".",":","!","?"]
parser = argparse.ArgumentParser(description='blah')
parser.add_argument('--input_file', type=str, help='the script')
args = parser.parse_args()
INPUT_FILE = args.input_file
f = open(INPUT_FILE+".txt","r+")
originalScript = f.read()
f.close()
f = open(INPUT_FILE+".json","r+")
fileData = f.read()
f.close()
data = json.loads(fileData)
WORD_COUNT = len(data['words'])
pose = -1
prevPose = -1
prevPhoneme = "na"
emotion = "0"
pararaph = 0
image = 0
OS_IndexAt = 0
pickNewPose(0)
strings[1] += "0,emotion,0\n"
strings[0] += "0,paragraph,0\n"
strings[2] += "0,image,0\n"
strings[4] += "0,phoneme,m\n"
for i in range(WORD_COUNT):
word = data['words'][i]
if "start" not in word:
continue
wordString = word["word"]
timeStart = word["start"]
OS_nextIndex = originalScript.index(wordString,OS_IndexAt)+len(wordString)
if "<" in originalScript[OS_IndexAt:]:
tagStart = originalScript.index("<",OS_IndexAt)
tagEnd = originalScript.index(">",OS_IndexAt)
if OS_nextIndex > tagStart and tagEnd >= OS_nextIndex:
OS_nextIndex = originalScript.index(wordString,tagEnd)+len(wordString)
nextDigest = originalScript[OS_IndexAt:OS_nextIndex]
if "\n" in nextDigest and data['words'][i-1]['case'] != 'not-found-in-audio' and (prevPhoneme == "a" or prevPhoneme == "f" or prevPhoneme == "u" or prevPhoneme == "y"):
addPhoneme("m", data['words'][i-1]["end"])
"""print(wordString)
print(str(OS_IndexAt)+", "+str(OS_nextIndex))
print(nextDigest)
print("")"""
pickedPose = False
for stopper in STOPPERS:
if stopper in nextDigest:
pickNewPose(timeStart)
pickedPose = True
if "<" in nextDigest:
leftIndex = nextDigest.index("<")+1
rightIndex = nextDigest.index(">")
emotion = emotions[nextDigest[leftIndex:rightIndex]]
strings[1] += (str.format('{0:.3f}', timeStart)+",emotion,"+str(emotion)+"\n")
prevPhoneme = "na"
if "\n\n" in nextDigest:
pararaph += 1
image += 1 # The line of the script advances 2 lines whenever we hit a /n/n.
strings[0] += (str.format('{0:.3f}', timeStart)+",paragraph,"+str(pararaph)+"\n")
prevPhoneme = "na"
if "\n" in nextDigest:
image += 1
strings[2] += (str.format('{0:.3f}', timeStart)+",image,"+str(image)+"\n")
prevPhoneme = "na"
if not pickedPose:
pickNewPose(timeStart) # A new image means we also need to have a new pose
phones = word["phones"]
timeAt = timeStart
for phone in phones:
timeAt += phone["duration"]
phoneString = phone["phone"]
if phoneString == "sil":
truePhone = "m"
else:
truePhone = mouths[phoneString[:phoneString.index("_")]]
if len(truePhone) == 2:
addPhoneme(truePhone[0], timeAt-phone["duration"])
addPhoneme(truePhone[1], timeAt-phone["duration"]*0.5)
else:
addPhoneme(truePhone, timeAt-phone["duration"])
OS_IndexAt = OS_nextIndex
f = open(INPUT_FILE+"_schedule.csv","w+")
for i in range(len(strings)):
f.write(strings[i])
if i < len(strings)-1:
f.write("SECTION\n")
f.flush()
f.close()
print(f"Done creating schedule for {INPUT_FILE}.")

The
ValueError: substring not found
occurs when you try to find the index of a substring in a string which does not contain it in the specified (or default) section, using the index function.
The index method takes 3 parameters:
value
start
end
and it searches for the value between start and end.
So, the error occurred because the substring was not found in the section where it was searched for. The line of
OS_nextIndex = originalScript.index(wordString,tagEnd)+len(wordString)
searches for wordString, starting from tagEnd and searches for the likes of
<span>yourwordstring</span>
, but in your case it was not found. You can do one of the following to solve the issue:
you can fix your input if it should always have a match for the search
you can handle the error when the index throws the error
you can use find instead, see https://bobbyhadz.com/blog/python-valueerror-substring-not-found
Note that find also has three parameters, as you can read from https://www.w3schools.com/python/ref_string_find.asp

Program does not print the return value of a function

I have a piece of code to calculate one matrix from another. But when I try to print the output of the function the code runs without any errors but does not print anything. Furthermore, if I try to print the input matrix to the function and then call the function, the original input does not get printed. It prints fine when I don't call this function inf_mat. Does anybody know what might be the reason?
import numpy as np
from itertools import permutations
def inf_mat(adjacency, l_max):
## Influence matrix - Returns the influence matrix corresponding to the input adjacency matrix
inf_arr = np.zeros(np.shape(adjacency))
print(inf_arr)
norm_mat = adjacency
for i in range(np.shape(norm_mat)[0]):
for j in range(np.shape(norm_mat)[1]):
if norm_mat[i,j] != 0:
norm_mat[i,j] = 1
for i in range(np.shape(inf_arr)[0]):
for j in range(np.shape(inf_arr)[1]):
ind_lists = []
for k in range(l_max-1):
all_ind = [m for m in range(np.shape(adjacency)[1])]
all_ind.remove(i)
if j != i:
all_ind.remove(j)
ind_lists.append(list(permutations(all_ind, k+1)))
pairwise_list = []
for p in range(len(ind_lists)):
pairs = []
for q in range(len(ind_lists[p])):
mult_list = [i]
for r in range(len(ind_lists[p][q])):
mult_list.append(ind_lists[p][q][r])
mult_list.append(j)
pairs.append(mult_list)
pairwise_list.append(pairs)
inf_sum = adjacency[i,j]
norm_sum = norm_mat[i,j]
for t in range(len(pairwise_list)):
for t_ind in range(len(pairwise_list[t])):
inf_ele = 1
norm_ele = 1
for s in range(len(pairwise_list[t_ind])-1):
inf_ele *= adjacency[s,s+1]
norm_ele *= norm_mat[s,s+1]
inf_sum += inf_ele
norm_sum += norm_ele
inf_arr[i,j] = inf_sum/norm_sum
return inf_arr/l_max
def readFile(Filename):
fileObj = open(Filename, "r")
words = fileObj.read().splitlines()
fileObj.close()
return words
topo_list = readFile("EMT_RACIPE.topo")
topo_list.remove(topo_list[0])
gene_list = []
for i in range(len(topo_list)):
topo_list[i] = topo_list[i].split()
for j in range(2):
if gene_list.count(topo_list[i][j]) == 0:
gene_list.append(topo_list[i][j])
adjacency = np.zeros(shape = (len(gene_list),len(gene_list)))
for i in range(len(gene_list)):
indices = []
for j in range(len(topo_list)):
if topo_list[j].count(gene_list[i]) > 0:
indices.append(j)
for k in range(len(indices)):
if topo_list[indices[k]][0] == gene_list[i]:
if topo_list[indices[k]][2] == '1':
adjacency[i, gene_list.index(topo_list[indices[k]][1])] = 1
if topo_list[indices[k]][2] == '2':
adjacency[i, gene_list.index(topo_list[indices[k]][1])] = -1
if topo_list[indices[k]][1] == gene_list[i]:
if topo_list[indices[k]][2] == '1':
adjacency[gene_list.index(topo_list[indices[k]][0]), i] = 1
if topo_list[indices[k]][2] == '2':
adjacency[gene_list.index(topo_list[indices[k]][0]), i] = -1
N22E82 = adjacency
print(N22E82)
inf_N22E82 = inf_mat(N22E82, 10)
print(inf_N22E82)

list item to integer conversion triggers syntax error in python3

import re
import socket
import sys
def Check(ip,port):
try:
s = socket.socket()
s.settimeout(0.3)
s.connect((ip,port))
return s.recv(512)
except:
pass
def Scan():
start = sys.argv[1]
end = sys.argv[2]
endip = end.partition('.')
currentip = start.split('.')
while not (currentip == endip):
targetip = currentip[0]+"."+currentip[1]+"."+currentip[2]+"."+currentip[3]
print("Checking: "+targetip+"\n")
result = Check(targetip,21)
if result:
if re.search("FTP",result.decode('utf-8')):
retard = open('ftps.txt','a')
retard.write(targetip+"\n")
retard.close()
if (int(currentip[3]) < 255) and (int(currentip[0]) != int(endip[0])) and (int(currentip[1]) != int(endip[1])) and (int(currentip[2]) != int(endip[2])) and (int(currentip[3]) != int(endip[3])+1):
int(currentip[3]) += 1
elif (int(currentip[3]) == 255) and (int(currentip[0]) != int(endip[0])) and (int(currentip[1]) != int(endip[1])) and (int(currentip[2]) != int(endip[2])) and (int(currentip[3]) != int(endip[3])+1):
if (int(currentip[2]) < 255):
int(currentip[2]) += 1
int(currentip[3]) = 0
elif (int(currentip[2]) == 255):
if (int(currentip[1]) < 255):
int(currentip[1]) += 1
int(currentip[2]) = 0
int(currentip[3]) = 0
elif (int(currentip[0]) < int(endip[0])) and (int(currentip[0]) != 255) and (int(currentip[1]) == 255):
int(currentip[0]) += 1
int(currentip[1]) = 0
int(currentip[2]) = 0
int(currentip[3]) = 0
Scan()
int(currentip[0]) += 1 causes an error while the conversion of other items that are exactly the same way converted trigger none.
File "ftpscan.py", line 46
int(currentip[0]) += 1
^
SyntaxError: cannot assign to function call

Basically, i += 1, is same as i = i + 1
in your case
int(currentip[0]) += 1 , is same as int(currentip[0]) = int(currentip[0]) + 1
So, technically you can't assign to the function call.
Instead, this should work
currentip[0] = int(currentip[0]) + 1

You are trying to increment a number (what int() returns) as opposed to the contents of a variable.

"""
FTPScan by StYl3z
Greetz fly out to:
L0rd,Legolas,Prometheus,Smoky-Ice,izibitzi,Waterb0ng,MaXtOr
usage: python3 ftpscan.py <startip> <endip>
"""
import re
import socket
import sys
def Check(ip,port):
try:
s = socket.socket()
s.settimeout(0.3)
s.connect((ip,port))
return s.recv(512)
except:
pass
def Scan():
start = sys.argv[1]
end = sys.argv[2]
endip = end.split('.')
currentip = start.split('.')
while not (currentip == endip):
targetip = currentip[0]+"."+currentip[1]+"."+currentip[2]+"."+currentip[3]
print("Checking: "+targetip+"\n")
result = Check(targetip,21)
if result == 0:
if re.search("FTP",result.decode('utf-8')):
retard = open('ftps.txt','a')
retard.write(targetip+"\n")
retard.close()
if not (int(currentip[3])==255):
currentip[3] = int(currentip[3])+1
currentip[3] = str(currentip[3])
else:
if not(int(currentip[2])==255):
currentip[2] = int(currentip[2])+1
currentip[2] = str(currentip[2])
currentip[3] = str("0")
else:
if not(int(currentip[1])==255):
currentip[1] = int(currentip[1])+1
currentip[1] = str(currentip[1])
currentip[2] = str("0")
currentip[3] = str("0")
else:
if not(int(currentip[0])==255):
currentip[0] = int(currentip[0])+1
currentip[0] = str(currentip[0])
currentip[1] = str("0")
currentip[2] = str("0")
currentip[3] = str("0")
Scan()
is the way it's working, thanks for trying to help me, i figured it out myself

Undefinded Syntax Error?

Very new to this, but so far have had a little help in making this. It keeps giving syntax error on line 131.. and many other errors too. Im not really sure on what it is and the person who helped me create this is gone leaving me by my noob self to try and fix this.There is more to the script in the middle after -------- which decompiles the direct files but i cant post them. But i only need to figure out whats wrong with this "syntax error" so, Whats worng with this script?
#o-----------------------------------------------------------------------------o
#(-----------------------------------------------------------------------------)
#|Compatible w/ Python 3.2 |
#########################
# Configuration section #
#########################
toptablefolder = 'C:/Games/TOP/kop/scripts/table/'
toptableversion = 6
toptableencoding = 'gbk'
####end of config####
import struct
from types import *
#---------------------------------------------
class top_tsv:
name = ''
file_name = ''
list = []
version = ''
frombin_map = [('',{'v':0})]
def __init__(self, file_name=0, version=0):
if file_name == 0:
self.file_name = toptablefolder+self.__class__.__name__
else:
self.file_name = file_name
if (version == 0):
self.version = toptableversion
else:
self.version = version
self.list = []
self.frombin_map = [('',{'v':0})]
def unencrypt(self):
item = []
#Load data
file = open(self.file_name+'.bin', 'rb')
data = bytearray(file.read())
file.close()
i = 0
array_size = int(list(struct.unpack_from('<i',data[i:i+4]))[0])
rdata = data[i:i+4]
data = data[i+4:len(data)]
m = 0
k = 0
l = len(data)//array_size
#2.2 encryption
if ((self.version >= 5) and ((self.__class__.__name__ != 'magicsingleinfo') and (self.__class__.__name__ != 'magicgroupinfo') and (self.__class__.__name__ != 'resourceinfo') and (self.__class__.__name__ != 'terraininfo'))):
newbytes = bytes.fromhex('98 9D 9F 68 E0 66 AB 70 E9 D1 E0 E0 CB DD D1 CB D5 CF')
while(k<l):
item = data[i:i+array_size]
i = i+array_size
if ((self.version >= 5) and ((self.__class__.__name__ != 'magicsingleinfo') and (self.__class__.__name__ != 'magicgroupinfo') and (self.__class__.__name__ != 'resourceinfo') and (self.__class__.__name__ != 'terraininfo'))):
for m,c in enumerate(item):
j = m % len(newbytes)
item[m] = ((item[m] - newbytes[j]) + 256) % 256
rdata = rdata + item
k = k + 1
m = 0
file = open(self.file_name+'-un.bin', 'wb')
file.write(rdata)
file.close()
def load_bin_data(self):
array = []
item = []
addresses = []
#Load structure (calculate size)
struct_type_map={
"char":"c",
"byte":"b",
"ubyte":"B",
"_bool":"?",
"short":"h",
"ushort":"H",
"int":"i",
"uint":"I",
"long":"l",
"ulong":"L",
"quad":"q",
"uquad":"Q",
"float":"f",
"double":"d",
"str":"s",
"color":"B",
"rcolor":"B",
}
struct_vars = {'t':'','s':1,'l':1,'stp':1,'f':-1,'v':-1,'lpad':0,'rpad':0,'sa':-1,'sap':-1,'ea':-1,'eap':-1,'st':'','lm':0,'sm':0,'smb':0,'smea':0,'sv':0,'func':0}
#initialize addresses
struct_init_address = 0
struct_limit_address = 0
struct_marked_addresses = [0,0,0,0,0,0]
struct_func_indexes = []
for i, v in enumerate(list(zip(*self.frombin_map))[1]):
struct_item = struct_vars.copy()
struct_item.update(v)
if struct_item['smb']>=1:
struct_marked_addresses[struct_item['smb']] = struct_init_address
if struct_item['lm']>=1:
struct_init_address = struct_marked_addresses[struct_item['lm']]
if struct_item['sm']>=1:
struct_marked_addresses[struct_item['sm']] = struct_init_address
if (type(struct_item['func']) == FunctionType):
struct_func_indexes.append(i)
elif (struct_item['v'] == -1):
if type(struct_item['t']) == tuple:
struct_item['s'] = len(struct_item['t'])
struct_item['st'] = []
for j,t in enumerate(struct_item['t']):
struct_item['st'].append(struct_type_map[struct_item['t'][j]])
struct_item['st'] = tuple(struct_item['st'])
struct_item['s'] = len(struct_item['st'])
else:
struct_item['st'] = struct_type_map[struct_item['t']]
if ((struct_item['t'] == 'color') or (struct_item['t'] == 'rcolor')):
struct_item['s'] = 3
struct_item['sa'] = struct_init_address
struct_item['sap'] = struct_item['sa'] + struct_item['lpad']
struct_item['ea'] = struct_item['sap']
if type(struct_item['t']) == tuple:
for j,t in enumerate(struct_item['t']):
struct_item['ea'] = struct_item['ea'] + struct.calcsize(struct_item['st'][j]) * ((struct_item['stp'] * struct_item['l'])-(struct_item['stp']-1))
else:
struct_item['ea'] = struct_item['ea'] + struct.calcsize(struct_item['st']) * ((struct_item['s'] * struct_item['stp'] * struct_item['l'])-(struct_item['stp']-1))
if struct_item['smea']>=1:
struct_marked_addresses[struct_item['smea']] = struct_item['ea']
struct_item['eap'] = struct_item['ea'] + struct_item['rpad']
struct_init_address = struct_item['eap']
if (struct_init_address > struct_limit_address):
struct_limit_address = struct_init_address
#print(struct_item)
self.frombin_map[i] = (self.frombin_map[i][0],struct_item)
struct_size = struct_limit_address
#Load data
file = open(self.file_name+'.bin', 'rb')
data = bytearray(file.read())
file.close()
i = 0
k = 0
array_size = int(list(struct.unpack_from('<i',data[i:i+4]))[0])
if array_size != struct_size:
print(self.file_name+'.bin: Actual array size ('+str(array_size)+') doesn''t match structure size ('+str(struct_size)+')')
raise 'Size error.'
data = data[i+4:len(data)]
m = 0
k = 0
l = len(data)//struct_size
#2.2 encryption
if ((self.version >= 5) and ((self.__class__.__name__ != 'magicsingleinfo') and (self.__class__.__name__ != 'magicgroupinfo') and (self.__class__.__name__ != 'resourceinfo') and (self.__class__.__name__ != 'terraininfo'))):
newbytes = bytes.fromhex('98 9D 9F 68 E0 66 AB 70 E9 D1 E0 E0 CB DD D1 CB D5 CF')
while(k<l):
item = data[i:i+struct_size]
i = i+struct_size
if ((self.version >= 5) and ((self.__class__.__name__ != 'magicsingleinfo') and (self.__class__.__name__ != 'magicgroupinfo') and (self.__class__.__name__ != 'resourceinfo') and (self.__class__.__name__ != 'terraininfo'))):
for m,c in enumerate(item):
j = m % len(newbytes)
item[m] = ((item[m] - newbytes[j]) + 256) % 256
array.append(item)
k = k + 1
m = 0
#Associate the data with the structure
self.list = []
self.list.append(list(zip(*self.frombin_map))[0])
for y,rawrow in enumerate(array):
row = []
for x,cell_struct in enumerate(list(zip(*self.frombin_map))[1]):
if type(cell_struct['func']) == FunctionType:
cell = []
cell.append('0')
row.append(self.transformtostr(cell,**cell_struct))
else:
cell = []
if (cell_struct['v'] == -1):
i = cell_struct['sap']
for j in range(cell_struct['l']):
processed_data=list(struct.unpack_from('<'+str((cell_struct['s']//len(cell_struct['st']))*cell_struct['stp'])+"".join(cell_struct['st']),rawrow,i))[::cell_struct['stp']]
#if (x == 4) and (y == 70):
# #print(cell_struct['s'])
cell.append(processed_data)
i=i+cell_struct['s']*struct.calcsize("".join(cell_struct['st']))*cell_struct['stp'] #sizeof here
if (cell_struct['t'] == 'rcolor'):
cell[0].reverse()
else:
cell.append(cell_struct['v'])
#if y == 70:
# print(cell) #'s':0x02,'l':0x08
row.append(self.transformtostr(cell,**cell_struct))
self.list.append(row)
for x,row in enumerate(self.list):
if x>0:
for func_index in struct_func_indexes:
self.list[x][func_index] = list(zip(*self.frombin_map))[1][func_index]['func'](func_index, row)
deletions = 0
for z,struct_item_name in enumerate(list(zip(*self.frombin_map))[0]):
if (struct_item_name == ''):
del self.list[x][z-deletions]
deletions = deletions + 1
return
def i(self,x):
for i,v in enumerate(self.list):
if(x==v):
return i
break
return -1
def j(self,x):
for i,v in enumerate(list(zip(*self.frombin_map))[0]):
if(x==v):
return i
break
return -1
def remap(self,v):
for i,n in enumerate(list(zip(*v))[0]):
if self.j(n) == -1:
self.frombin_map.append((list(zip(*v))[0][i],list(zip(*v))[1][i]))
else:
if (list(zip(*v))[1][i] == {}):
del self.frombin_map[self.j(n)]
else:
self.frombin_map[self.j(n)] = (list(zip(*v))[0][i],list(zip(*v))[1][i])
def bintotsv(self):
file = open(self.file_name+'.txt', 'wt', encoding=toptableencoding)
for i,a in enumerate(self.list):
a=list(a)
if (i==0):
deletions = 0
for z,item in enumerate(a[:]):
if (item == ''):
del a[z-deletions]
deletions = deletions + 1
file.write('//'+'\t'.join(a))
else:
#print(a)
file.write('\n'+'\t'.join(a))
file.close()
return self
def trim_nullbytestr(string, flag=0): #flag=1, return "0" when string is empty
result = string
for i,byte in enumerate(string[:]):
if byte == '\00':
result = string[:i]
break
if (flag & 1) and ((string == '\00') or (string == '')):
result = '0'
return result
def transformtostr(self,result,t,s,l,stp,f,v,ea,eap,lpad,rpad,sa,sap,st,sm,lm,smb,smea,sv,func):
if v != -1:
return str(v)
_type = t
j = 0
for n,v in enumerate(result[:]):
m = 0
if type(t) == tuple:
_type = t[j]
is_integer = not((_type=="float") or (_type=="double") or (_type=="str") or (_type=="char") or (_type=="_bool"))
if is_integer:
if f==-1:
f=0
for k,w in enumerate(v[:]):
if is_integer:
#Result: flag=0x0001 = remove FF's, #flag=0x0002 = cut file defect bytes, #flag=0x0004 = remove 0's
if ((((f & 0x4) and (w == 0))) or
((f & 0x1) and ((w == (2**8)-1) or (w == (2**16)-1) or (w == (2**32)-1) or (w == (2**64)-1) or (w == -1))) or
((f & 0x2) and ((((_type=="byte") or (_type=="ubyte")) and (w == (2**8)-51)) or (((_type=="short") or (_type=="ushort")) and (w == (2**16)-12851)) or (((_type=="long") or (_type=="ulong")) and (w == (2**32)-842150451)) or (((_type=="quad") or (_type=="uquad")) and (w == (2**64)-3617008641903833651))))):
del result[j][m]
m=m-1
else:
if (f & 0x2 and w == 0):
result[j] = result[j][:m]
break
result[j][m]=str(result[j][m])
m=m+1
if (_type=="float"):
if f==-1:
f=3
result[j][k]=str(round(float(w),f))
if (_type=="str"):
if f==-1:
f=0
if (w[0] == 205) and (w[1] == 205):
result[j][m] = ''
else:
result[j][m] = w.decode('gbk','ignore')
for o,x in enumerate(result[j][m]):
if x == '\00':
result[j][m] = result[j][m][:o]
break
#result[j][m] = result[j][m].strip()
if ((f & 1) and (result[j][m] == '')):
result = '0'
if (result[j] != '0'):
result[j] = ','.join(result[j])
if is_integer:
if (result[j] == ''):
result[j] = '0'
j=j+1
if (_type=="str"):
result = ','.join(result)
if ((f & 1) and (result == '')):
result = '0'
else:
result = ';'.join(result)
if is_integer:
if (result == ''):
result = '0'
return result
]
for c in toptablebins:
myc = c()
try:
myc.load_bin_data()
except IOError:
print('"'+myc.file_name+'.bin'+'" doesn''t exist, skipping...')
myc.bintotsv()
these are a few lines below and above line 131 which is the error
for j,t in enumerate(struct_item['t']):
struct_item['st'].append(struct_type_map[struct_item['t'][j]])
struct_item['st'] = tuple(struct_item['st'])
struct_item['s'] = len(struct_item['st'])
else: <--- this is 131
struct_item['st'] = struct_type_map[struct_item['t']]
if ((struct_item['t'] == 'color') or (struct_item['t'] == 'rcolor')):
struct_item['s'] = 3
struct_item['sa'] = struct_init_address
struct_item['sap'] = struct_item['sa'] + struct_item['lpad']
struct_item['ea'] = struct_item['sap']
if type(struct_item['t']) == tuple:
for j,t in enumerate(struct_item['t']):
struct_item['ea'] = struct_item['ea'] + struct.calcsize(struct_item['st'][j]) * ((struct_item['stp'] * struct_item['l'])-(struct_item['stp']-1))

It doesn't look like the you have the proper indention. Remember, in Python, whitespace is very very important.
The problem section of code should probably start looking something like:
elif (struct_item['v'] == -1):
if type(struct_item['t']) == tuple:
struct_item['s'] = len(struct_item['t'])
struct_item['st'] = []
for j,t in enumerate(struct_item['t']):
struct_item['st'].append(struct_type_map[struct_item['t'][j]])
struct_item['st'] = tuple(struct_item['st'])
struct_item['s'] = len(struct_item['st'])

This seems a bit weird:
l = len(data)//struct_size
You really need to post the traceback the the interpreter is giving you - it makes finding the error much easier.