'Python Ray has issues with ArcPy (ArcGIS Pro)
I'm working on a large-size Vehicle Routing Problems using ArcGIS. As my problem has large number of non-overlapping vehicle routing zones, I have decided to use Python multiprocessing library to speed up the analysis. But I've noticed the overhead cost is too high, even though I passed only text arguments to pool to minimize pickling efforts (outcome of the multiprocessing is also string only). A part of my code is like follows:
import arcpy
import multiprocessing
def vrp_mp(in_args):
sm = in_args['sm']
arcpy.env.workspace = in_args['workspace']
data_gdb = in_args['data_gdb']
orders_ly = arcpy.MakeFeatureLayer_management(data_gdb + '\\' + in_args['orders_fc'])
middle_zone_ly = arcpy.MakeFeatureLayer_management(data_gdb + '\\' + in_args['attend_middle_zone_fc'])
arcpy.SelectLayerByAttribute_management(middle_zone_ly, 'new_selection', f'RouteZone = \'{sm}\'')
arcpy.SelectLayerByLocation_management(orders_ly, 'intersect', middle_zone_ly)
selected_orders = f'memory\\selected_orders_{sm}'
selected_zone = f'memory\\selected_zone_{sm}'
selected_route_tb = f'memory\\selected_route_tb_{sm}'
arcpy.CopyFeatures_management(orders_ly, selected_orders)
arcpy.CopyFeatures_management(middle_zone_ly, selected_zone)
arcpy.TableSelect_analysis(data_gdb + '\\' + in_args['attend_route_tb'], selected_route_tb, f'"Name" = \'{sm}\'')
ND_data = arcpy.nax.MakeNetworkDatasetLayer(in_args['network_dataset'])
nd_travel_modes = arcpy.nax.GetTravelModes(ND_data)
travel_mode = list(nd_travel_modes.keys())[0]
vrp_result = vrp_inLoop(ND_data, travel_mode, in_args['default_date'], selected_orders,
data_gdb + '\\' + in_args['depot_fc'], selected_zone,
selected_route_tb, 'TimeWindowStart', 'TimeWindowEnd', in_args['search_query'], in_args['order_id_f'])
if vrp_result.solveSucceeded:
vrp_result.export(arcpy.nax.VehicleRoutingProblemOutputDataType2.Orders, arcpy.env.workspace + f'\\Orders_{sm}')
vrp_result.export(arcpy.nax.VehicleRoutingProblemOutputDataType2.Routes, arcpy.env.workspace + f'\\Routes_{sm}')
print(f'{sm} VRP 완료')
else:
solve_msgs = "\n".join([msg[-1] for msg in vrp_result.solverMessages(arcpy.nax.MessageSeverity.All)])
print(solve_msgs)
raise Exception
return [arcpy.env.workspace + f'\\Orders_{sm}', arcpy.env.workspace + f'\\Routes_{sm}']
order_fc = 'orders'
input_args = []
#attend_list is a list of names in string
#all variables in argument dictionary are string
input_args = []
for sm in attend:
a = {'network_dataset':network_dataset, 'default_date': default_date, 'orders_fc':orders_fc, 'depot_fc':depot_fc,
'attend_middle_zone_fc':attend_middle_zone_fc, 'attend_route_tb':attend_route_tb, 'TimeStart':"TimeWindowStart", 'TimeEnd': 'TimeWindowEnd',
'search_query':search_query, 'order_id_f':order_id_f, 'sm':sm, 'data_gdb':data_gdb, 'workspace': f'{path}vrp_work{attend.index(sm)}\\scratch.gdb'}
input_args.append(a)
work_size = min(len(input_args), multiprocessing.cpu_count())
pool = multiprocessing.Pool(processes=work_size)
results = pool.map(vrp_mp, input_args)
So I've decided to move to Ray library. However, I saw several issues when I tried to use ray for my job.
import ray
import arcpy
import time
import random as rd
arcpy.env.overwriteOutput = True
arcpy.CheckOutExtension("Network")
path = 'C:\\InsuHong\\RouteZone_Opt\\'
@ray.remote
def arc_test(pt_fc):
a = rd.randint(1, 9999999)
arcpy.MinimumBoundingGeometry_management( pt_fc, f'ch_{a}s', 'convex_hull')
orders_fc = 'orders'
middle_zone_fc = 'zone_4'
data_gdb = r'C:\InsuHong\NLMD\ray_test.gdb'
arcpy.env.workspace = data_gdb
ray.init(num_cpus=2)
route_tb = 'routes'
zone_ly = arcpy.MakeFeatureLayer_management(middle_zone_fc)
route_names = [row[0] for row in arcpy.da.SearchCursor(zone_ly, ['RouteName'])]
stime = time.time()
arc_test(orders_fc)
result = ray.get([arc_test.remote(orders_fc) for sm in [0,1]])
print(time.time()-stime)
- Windows fatal error occurred after initializing ray and running any function in ArcPy. Like below, even though the given Arcpy function is not a part of ray task, Windows fatal error always occurred. However, the code ran until the end, including the given Arcpy function, without any errors (I'm using Wing IDE and Anaconda Env).
Windows fatal exception: code 0xe0000001
Stack (most recent call first):
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 512 in <lambda>
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\management.py", line 8823 in MakeFeatureLayer
File "C:\InsuHong\VRS\VRS_repo\arc_ray_test.py", line 24 in <module>
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 3966 in __RunFileInMain
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 932 in RunFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\netserver.py", line 1062 in Run
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 459 in _DebugFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 690 in StartDebug
File "C:\Program Files (x86)\Wing Pro 8\bootstrap\wingdb.py", line 142 in <module>
Windows fatal exception: code 0xe0000001
Stack (most recent call first):
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 512 in <lambda>
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\management.py", line 8823 in MakeFeatureLayer
File "C:\InsuHong\VRS\VRS_repo\arc_ray_test.py", line 24 in <module>
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 3966 in __RunFileInMain
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 932 in RunFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\netserver.py", line 1062 in Run
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 459 in _DebugFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 690 in StartDebug
File "C:\Program Files (x86)\Wing Pro 8\bootstrap\wingdb.py", line 142 in <module>
Windows fatal exception: code 0xe0000001
Stack (most recent call first):
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 512 in <lambda>
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\management.py", line 8823 in MakeFeatureLayer
File "C:\InsuHong\VRS\VRS_repo\arc_ray_test.py", line 24 in <module>
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 3966 in __RunFileInMain
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 932 in RunFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\netserver.py", line 1062 in Run
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 459 in _DebugFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 690 in StartDebug
File "C:\Program Files (x86)\Wing Pro 8\bootstrap\wingdb.py", line 142 in <module>
Windows fatal exception: code 0xe0000001
Stack (most recent call first):
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 512 in <lambda>
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\management.py", line 8823 in MakeFeatureLayer
File "C:\InsuHong\VRS\VRS_repo\arc_ray_test.py", line 24 in <module>
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 3966 in __RunFileInMain
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 932 in RunFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\netserver.py", line 1062 in Run
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 459 in _DebugFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 690 in StartDebug
File "C:\Program Files (x86)\Wing Pro 8\bootstrap\wingdb.py", line 142 in <module>
Windows fatal exception: code 0xe0000001
Stack (most recent call first):
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 512 in <lambda>
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\management.py", line 8823 in MakeFeatureLayer
File "C:\InsuHong\VRS\VRS_repo\arc_ray_test.py", line 24 in <module>
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 3966 in __RunFileInMain
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 932 in RunFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\netserver.py", line 1062 in Run
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 459 in _DebugFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 690 in StartDebug
File "C:\Program Files (x86)\Wing Pro 8\bootstrap\wingdb.py", line 142 in <module>
Windows fatal exception: code 0xe0000001
Stack (most recent call first):
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 512 in <lambda>
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\management.py", line 8823 in MakeFeatureLayer
File "C:\InsuHong\VRS\VRS_repo\arc_ray_test.py", line 24 in <module>
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 3966 in __RunFileInMain
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 932 in RunFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\netserver.py", line 1062 in Run
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 459 in _DebugFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 690 in StartDebug
File "C:\Program Files (x86)\Wing Pro 8\bootstrap\wingdb.py", line 142 in <module>
Windows fatal exception: code 0xe0000001
Stack (most recent call first):
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 512 in <lambda>
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\management.py", line 8823 in MakeFeatureLayer
File "C:\InsuHong\VRS\VRS_repo\arc_ray_test.py", line 24 in <module>
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 3966 in __RunFileInMain
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\dbgserver.py", line 932 in RunFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\netserver.py", line 1062 in Run
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 459 in _DebugFile
File "C:\Program Files (x86)\Wing Pro 8\bin\dbg\src\debug\tserver\startdebug.py", line 690 in StartDebug
File "C:\Program Files (x86)\Wing Pro 8\bootstrap\wingdb.py", line 142 in <module>
- Arcpy function never worked inside task function, and I got a very long error message with broken lines, so I have no idea what was wrong. I tried to import Arcpy inside the task function, but it did not work. Even with the simplest function like MakeFeatureLayer, it's never worked. I assigned the workspace explicitly inside the function, and passed the exact location of the featureclass, but it did not work at all.
Following is a part of weird errors.
[2m[36m(pid=2388)[0m Stack (most recent call first):
[2m[36m(pid=2388)[0m File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 70 in __init__
[2m[36m(pid=2388)[0m File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 615 in <module>
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 219 in _call_with_frames_removed
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap_external>", line 728 in exec_module
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 677 in _load_unlocked
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 967 in _find_and_load_unlocked
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 983 in _find_and_load
[2m[36m(pid=2388)[0m File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\__init__.py", line 14 in <module>
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 219 in _call_with_frames_removed
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap_external>", line 728 in exec_module
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 677 in _load_unlocked
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 967 in _find_and_load_unlocked
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 983 in _find_and_load
[2m[36m(pid=2388)[0m File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\__init__.py", line 90 in <module>
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 219 in _call_with_frames_removed
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap_external>", line 728 in exec_module
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 677 in _load_unlocked
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 967 in _find_and_load_unlocked
[2m[36m(pid=2388)[0m File "<frozen importlib._bootstrap>", line 983 in _find_and_load
[2m[36m(pid=2388)[0m File "C:\Users\admin\anaconda3\envs\arcpy28\lib\site-packages\ray\cloudpickle\cloudpickle.py", line 679 in subimport
[2m[36m(pid=2388)[0m File "C:\Users\admin\anaconda3\envs\arcpy28\lib\site-packages\ray\_private\function_manager.py", line 240 in fetch_and_register_remote_function
[2m[36m(pid=2388)[0m File "C:\Users\admin\anaconda3\envs\arcpy28\lib\site-packages\ray\_private\import_thread.py", line 150 in _process_key
[2m[36m(pid=2388)[0m File "C:\Users\admin\anaconda3\envs\arcpy28\lib\site-packages\ray\_private\import_thread.py", line 96 in _do_importing
[2m[36m(pid=2388)[0m File "C:\Users\admin\anaconda3\envs\arcpy28\lib\site-packages\ray\_private\import_thread.py", line 81 in _run
[2m[36m(pid=2388)[0m File "C:\Users\admin\anaconda3\envs\arcpy28\lib\threading.py", line 870 in run
[2m[36m(pid=2388)[0m File "C:\Users\admin\anaconda3\envs\arcpy28\lib\threading.py", line 926 in _bootstrap_inner
[2m[36m(pid=2388)[0m File "C:\Users\admin\anaconda3\envs\arcpy28\lib\threading.py", line 890 in _bootstrap
[2m[36m(pid=2388)[0m Windows fatal exception: code 0xe0000001
And the following is IDE's debugger IO:
File "C:\InsuHong\VRS\VRS_repo\arc_ray_test.py", line 27, in <module>
result = ray.get([arc_test.remote(orders_fc) for sm in [0,1]])
File "C:\Users\admin\anaconda3\envs\arcpy28\Lib\site-packages\ray\_private\client_mode_hook.py", line 105, in wrapper
return func(*args, **kwargs)
File "C:\Users\admin\anaconda3\envs\arcpy28\Lib\site-packages\ray\worker.py", line 1733, in get
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(ExecuteError): [36mray::arc_test()[39m (pid=2388, ip=127.0.0.1)
File "python\ray\_raylet.pyx", line 636, in ray._raylet.execute_task
File "python\ray\_raylet.pyx", line 640, in ray._raylet.execute_task
File "C:\InsuHong\VRS\VRS_repo\arc_ray_test.py", line 15, in arc_test
arcpy.MinimumBoundingGeometry_management( pt_fc, f'ch_{a}s', 'convex_hull')
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\management.py", line 4208, in MinimumBoundingGeometry
raise e
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\management.py", line 4205, in MinimumBoundingGeometry
retval = convertArcObjectToPythonObject(gp.MinimumBoundingGeometry_management(*gp_fixargs((in_features, out_feature_class, geometry_type, group_option, group_field, mbg_fields_option), True)))
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 512, in <lambda>
return lambda *args: val(*gp_fixargs(args, True))
arcgisscripting.ExecuteError: Failed to execute. Parameters are not valid..
Failed to execute (MinimumBoundingGeometry).
FYI, there is no problem in the argument for arcpy.MinimumBoundingGeometry.
So I'm wondering what I have done wrong. If not, would it be the issue of ArcPy? I googled it, but I could not find any use case or questions about Python Ray + Arcpy.
How do I fix this problem?
Thanks in advance.
Insu
Solution 1:[1]
I have a similar problem trying to parallelize a process that calls a function of Arcpy using Spyder as an IDE. I have created this test functions that are simply using the arcpy.AddMessage() function as a test and my conde doesn't work :(
import arcpy
from arcpy.sa import *
import time
import ray
list_start_index =[]
list_end_index =[]
list_of_output_files=[]
location_of_stack_profiles =r'E:\example'
root_folder_results = r'E:\results'
root_name_result_files = r'\Example_parallel'
start_profile_task_1 = 1
end_profile_task_1 = 10
location_to_save_excel_file_task_1 = root_folder_results + root_name_result_files + str(start_profile_task_1) +'_to_'+ str(end_profile_task_1) + r'.xlsx'
list_start_index.append(start_profile_task_1)
list_end_index.append(end_profile_task_1)
list_of_output_files.append(location_to_save_excel_file_task_1)
start_profile_task_2 = 11
end_profile_task_2 = 20
location_to_save_excel_file_task_2 = root_folder_results + root_name_result_files + str(start_profile_task_2) +'_to_'+ str(end_profile_task_2) + r'.xlsx'
list_start_index.append(start_profile_task_2)
list_end_index.append(end_profile_task_2)
list_of_output_files.append(location_to_save_excel_file_task_2)
start_profile_task_3 = 21
end_profile_task_3 = 30
location_to_save_excel_file_task_3 = root_folder_results + root_name_result_files + str(start_profile_task_3) +'_to_'+ str(end_profile_task_3) + r'.xlsx'
list_start_index.append(start_profile_task_3)
list_end_index.append(end_profile_task_3)
list_of_output_files.append(location_to_save_excel_file_task_3)
ray.shutdown()
ray.init()
@ray.remote
def test_function_1(Start_index,End_index,List_Tables,location_of_stack_profiles,location_to_save_excel_file):
print('This is a test')
arcpy.AddMessage('This is a test')
a=location_to_save_excel_file
return a
@ray.remote
def test_function_2(Start_index,End_index,List_Tables,location_of_stack_profiles,location_to_save_excel_file):
print('This is a test again')
arcpy.AddMessage('This is a test again')
b=location_to_save_excel_file
return b
@ray.remote
def test_function_3(Start_index,End_index,List_Tables,location_of_stack_profiles,location_to_save_excel_file):
print('This is a test again agian ')
arcpy.AddMessage('This is a test again agian ')
c=location_to_save_excel_file
return c
results=[]
results= ray.get([test_function_1.remote(list_start_index[0],list_end_index[0],List_Tables,location_of_stack_profiles,list_of_output_files[0]),
test_function_2.remote(list_start_index[1],list_end_index[1],List_Tables,location_of_stack_profiles,list_of_output_files[1]),
test_function_3.remote(list_start_index[2],list_end_index[2],List_Tables,location_of_stack_profiles,list_of_output_files[2])])
the result in console are a bunch of broken lines that look like this:
(pid=12244) File "<frozen importlib._bootstrap>", line 967 in _find_and_load_unlocked
.........
(pid=7472) e\
(pid=7472) c
(pid=7472) lo
(pid=7472) ud
(pid=7472) p
(pid=7472) i
(pid=7472) ck
(pid=7472) le
(pid=7472) .p
(pid=7472) y
(pid=7472) "
(pid=7472) , line 679
(pid=7472) in s
(pid=7472) u
(pid=7472) b
(pid=7472) im
(pid=7472) p
(pid=7472) o
(pid=7472) r
(pid=7472) t
(pid=7472)
(pid=7472) File
(pid=7472) "C
(pid=7472) :\U
(pid=7472) s
(pid=7472) er
(pid=7472) s\
(pid=7472) uq
(pid=7472) gp
(pid=7472) e
(pid=7472) r
(pid=7472) e
(pid=7472) 2
(pid=7472) \
(pid=7472) A
(pid=7472) p
(pid=7472) p
(pid=7472) D
(pid=7472) a
(pid=7472) t
(pid=7472) a\
(pid=7472) Roa
(pid=7472) m
(pid=7472) i
(pid=7472) ng
(pid=7472) \
(pid=7472) P
(pid=7472) yth
(pid=7472) o
(pid=7472) n\
(pid=7472) Py
(pid=7472) th
(pid=7472) on
(pid=7472) 37\
(pid=7472) si
(pid=7472) t
(pid=7472) e
(pid=7472) -
(pid=7472) pa
(pid=7472) c
(pid=7472) k
(pid=7472) a
(pid=7472) g
(pid=7472) e
(pid=7472) s
(pid=7472) \
(pid=7472) r
(pid=7472) a
(pid=7472) y
(pid=7472) \_
(pid=7472) p
(pid=7472) r
(pid=7472) i
(pid=7472) v
(pid=7472) a
(pid=7472) t
(pid=7472) e
(pid=7472) \
(pid=7472) f
(pid=7472) u
(pid=7472) n
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|---|
| Solution 1 | Gabriel Perez |