Coverage for kea/device_state.py: 69%

1import copy 

2import json 

3import logging 

4import math 

5import os 

6import random 

7 

8from typing import TYPE_CHECKING 

9if TYPE_CHECKING: 

10 from .device import Device 

11 

12from .utils import md5, deprecated 

13from .input_event import SearchEvent, SetTextAndSearchEvent, TouchEvent, LongTouchEvent, ScrollEvent, SetTextEvent, KeyEvent, UIEvent 

14 

15 

16class DeviceState(object): 

17 """ 

18 the state of the current device 

19 """ 

20 

21 def __init__( 

22 self, 

23 device:"Device", 

24 views, 

25 foreground_activity, 

26 activity_stack, 

27 background_services, 

28 tag=None, 

29 screenshot_path=None, 

30 ): 

31 self.device = device 

32 self.foreground_activity = foreground_activity 

33 self.activity_stack = activity_stack if isinstance(activity_stack, list) else [] 

34 self.background_services = background_services 

35 if tag is None: 

36 from datetime import datetime 

37 

38 tag = datetime.now().strftime("%Y-%m-%d_%H%M%S") 

39 self.tag = tag 

40 self.screenshot_path = screenshot_path 

41 if foreground_activity is not None: 

42 self.views = self.__parse_views(views) 

43 self.view_tree = {} 

44 self.__assemble_view_tree(self.view_tree, self.views) 

45 self.__generate_view_strs() 

46 self.state_str = self.__get_state_str() 

47 self.structure_str = self.__get_content_free_state_str() 

48 self.search_content = self.__get_search_content() 

49 self.text_representation = self.get_text_representation() 

50 else: 

51 self.views = [] 

52 self.view_tree = {} 

53 self.state_str = "home_page_or_lock_screen" 

54 self.structure_str = "home_page_or_lock_screen" 

55 self.search_content = "home_page_or_lock_screen" 

56 self.text_representation = "home_page_or_lock_screen" 

57 self.possible_events = None 

58 self.width = device.get_width(refresh=True) 

59 self.height = device.get_height(refresh=False) 

60 self.pagePath = self.__get_pagePath() 

61 self.logger = logging.getLogger(self.__class__.__name__) 

62 

63 @property 

64 def activity_short_name(self): 

65 return self.foreground_activity.split('.')[-1] 

66 

67 def to_dict(self): 

68 state = { 

69 'tag': self.tag, 

70 'state_str': self.state_str, 

71 'state_str_content_free': self.structure_str, 

72 'foreground_activity': self.foreground_activity, 

73 'activity_stack': self.activity_stack, 

74 'background_services': self.background_services, 

75 'width': self.width, 

76 'height': self.height, 

77 'views': self.views, 

78 } 

79 return state 

80 

81 def to_json(self): 

82 import json 

83 

84 return json.dumps(self.to_dict(), indent=2) 

85 

86 def __parse_views(self, raw_views): 

87 views = [] 

88 if not raw_views or len(raw_views) == 0: 

89 return views 

90 

91 for view_dict in raw_views: 

92 # # Simplify resource_id 

93 # resource_id = view_dict['resource_id'] 

94 # if resource_id is not None and ":" in resource_id: 

95 # resource_id = resource_id[(resource_id.find(":") + 1):] 

96 # view_dict['resource_id'] = resource_id 

97 views.append(view_dict) 

98 return views 

99 

100 def __assemble_view_tree(self, root_view, views): 

101 if not len(self.view_tree): # bootstrap 

102 if not len(views): 

103 return 

104 self.view_tree = copy.deepcopy(views[0]) 

105 self.__assemble_view_tree(self.view_tree, views) 

106 else: 

107 children = list(enumerate(root_view["children"])) 

108 if not len(children): 

109 return 

110 for i, j in children: 

111 root_view["children"][i] = copy.deepcopy(self.views[j]) 

112 self.__assemble_view_tree(root_view["children"][i], views) 

113 

114 def __generate_view_strs(self): 

115 for view_dict in self.views: 

116 self.__get_view_str(view_dict) 

117 # self.__get_view_structure(view_dict) 

118 

119 @staticmethod 

120 def __calculate_depth(views): 

121 root_view = None 

122 for view in views: 

123 if DeviceState.__safe_dict_get(view, 'parent') == -1: 

124 root_view = view 

125 break 

126 DeviceState.__assign_depth(views, root_view, 0) 

127 

128 @staticmethod 

129 def __assign_depth(views, view_dict, depth): 

130 view_dict['depth'] = depth 

131 for view_id in DeviceState.__safe_dict_get(view_dict, 'children', []): 

132 DeviceState.__assign_depth(views, views[view_id], depth + 1) 

133 

134 def __get_pagePath(self): 

135 for view in self.views: 

136 pagePath = self.__safe_dict_get(view, "pagePath") 

137 if pagePath: 

138 return pagePath 

139 

140 def __get_state_str(self): 

141 state_str_raw = self.__get_state_str_raw() 

142 return md5(state_str_raw) 

143 

144 def __get_state_str_raw(self): 

145 if self.device.humanoid is not None: 

146 import json 

147 from xmlrpc.client import ServerProxy 

148 

149 proxy = ServerProxy("http://%s/" % self.device.humanoid) 

150 return proxy.render_view_tree( 

151 json.dumps( 

152 { 

153 "view_tree": self.view_tree, 

154 "screen_res": [ 

155 self.device.display_info["width"], 

156 self.device.display_info["height"], 

157 ], 

158 } 

159 ) 

160 ) 

161 else: 

162 view_signatures = set() 

163 for view in self.views: 

164 if self.device.is_harmonyos: 

165 # exclude the com.ohos.sceneboard package in harmonyOS 

166 if self.__safe_dict_get(view, "package") == "com.ohos.sceneboard": 

167 continue 

168 view_signature = DeviceState.__get_view_signature(view) 

169 if view_signature: 

170 view_signatures.add(view_signature) 

171 return "%s{%s}" % (self.foreground_activity, ",".join(sorted(view_signatures))) 

172 

173 def __get_content_free_state_str(self): 

174 if self.device.humanoid is not None: 

175 import json 

176 from xmlrpc.client import ServerProxy 

177 

178 proxy = ServerProxy("http://%s/" % self.device.humanoid) 

179 state_str = proxy.render_content_free_view_tree( 

180 json.dumps( 

181 { 

182 "view_tree": self.view_tree, 

183 "screen_res": [ 

184 self.device.display_info["width"], 

185 self.device.display_info["height"], 

186 ], 

187 } 

188 ) 

189 ) 

190 else: 

191 view_signatures = set() 

192 

193 if self.activity_short_name == "DeckPicker": 

194 view_signatures = list() 

195 for view in self.views: 

196 view_signature = DeviceState.__get_content_free_view_signature(view) 

197 if view_signature: 

198 view_signatures.append(view_signature) 

199 else: 

200 for view in self.views: 

201 view_signature = DeviceState.__get_content_free_view_signature(view) 

202 if view_signature: 

203 view_signatures.add(view_signature) 

204 state_str = "%s{%s}" % ( 

205 self.foreground_activity, 

206 ",".join(sorted(view_signatures)), 

207 ) 

208 import hashlib 

209 

210 return hashlib.md5(state_str.encode('utf-8')).hexdigest() 

211 

212 def __get_search_content(self): 

213 """ 

214 get a text for searching the state 

215 :return: str 

216 """ 

217 words = [ 

218 ",".join(self.__get_property_from_all_views("resource_id")), 

219 ",".join(self.__get_property_from_all_views("text")), 

220 ] 

221 return "\n".join(words) 

222 

223 def __get_property_from_all_views(self, property_name): 

224 """ 

225 get the values of a property from all views 

226 :return: a list of property values 

227 """ 

228 property_values = set() 

229 for view in self.views: 

230 property_value = DeviceState.__safe_dict_get(view, property_name, None) 

231 if property_value: 

232 property_values.add(property_value) 

233 return property_values 

234 

235 def draw_event(self, event, screenshot_path): 

236 import cv2 

237 image = cv2.imread(screenshot_path) 

238 if event is not None and screenshot_path is not None: 

239 if isinstance(event, TouchEvent): 

240 cv2.rectangle(image, (int(event.view['bounds'][0][0]),int(event.view['bounds'][0][1])),(int(event.view['bounds'][1][0]),int(event.view['bounds'][1][1])), (0, 0, 255), 5) 

241 elif isinstance(event, LongTouchEvent): 

242 cv2.rectangle(image, (int(event.view['bounds'][0][0]),int(event.view['bounds'][0][1])),(int(event.view['bounds'][1][0]),int(event.view['bounds'][1][1])), (0, 255, 0), 5) 

243 elif isinstance(event, SetTextEvent): 

244 cv2.rectangle(image, (int(event.view['bounds'][0][0]),int(event.view['bounds'][0][1])),(int(event.view['bounds'][1][0]),int(event.view['bounds'][1][1])), (255, 0, 0), 5) 

245 elif isinstance(event, ScrollEvent): 

246 cv2.rectangle(image, (int(event.view['bounds'][0][0]),int(event.view['bounds'][0][1])),(int(event.view['bounds'][1][0]),int(event.view['bounds'][1][1])), (255, 255, 0), 5) 

247 elif isinstance(event, KeyEvent): 

248 cv2.putText(image,event.name, (100,300), cv2.FONT_HERSHEY_SIMPLEX, 5,(0, 255, 0), 3, cv2.LINE_AA) 

249 else: 

250 return 

251 try: 

252 cv2.imwrite(screenshot_path, image) 

253 except Exception as e: 

254 self.logger.warning(e) 

255 def save_view_img(self, view_dict, output_dir=None): 

256 try: 

257 if output_dir is None: 

258 if self.device.output_dir is None: 

259 return 

260 else: 

261 output_dir = os.path.join(self.device.output_dir, "views") 

262 if not os.path.exists(output_dir): 

263 os.makedirs(output_dir) 

264 view_str = view_dict['view_str'] 

265 if not self.device.is_harmonyos: 

266 if self.device.adapters[self.device.minicap]: 

267 view_file_path = "%s/view_%s.jpg" % (output_dir, view_str) 

268 else: 

269 view_file_path = "%s/view_%s.png" % (output_dir, view_str) 

270 else: 

271 # HarmonyOS 

272 view_file_path = "%s/view_%s.jpeg" % (output_dir, view_str) 

273 if os.path.exists(view_file_path): 

274 return 

275 from PIL import Image 

276 

277 # Load the original image: 

278 view_bound = view_dict['bounds'] 

279 original_img = Image.open(self.screenshot_path) 

280 # view bound should be in original image bound 

281 view_img = original_img.crop( 

282 ( 

283 min(original_img.width - 1, max(0, view_bound[0][0])), 

284 min(original_img.height - 1, max(0, view_bound[0][1])), 

285 min(original_img.width, max(0, view_bound[1][0])), 

286 min(original_img.height, max(0, view_bound[1][1])), 

287 ) 

288 ) 

289 view_img.convert("RGB").save(view_file_path) 

290 except Exception as e: 

291 self.device.logger.warning(e) 

292 

293 def is_different_from(self, another_state): 

294 """ 

295 compare this state with another 

296 @param another_state: DeviceState 

297 @return: boolean, true if this state is different from other_state 

298 """ 

299 return self.state_str != another_state.state_str 

300 

301 @staticmethod 

302 def __get_view_signature(view_dict): 

303 """ 

304 get the signature of the given view 

305 @param view_dict: dict, an element of list DeviceState.views 

306 @return: 

307 """ 

308 if 'signature' in view_dict: 

309 return view_dict['signature'] 

310 

311 view_text = DeviceState.__safe_dict_get(view_dict, 'text', "None") 

312 

313 if view_text is None or len(view_text) > 50 or DeviceState.__safe_dict_get(view_dict, 'class', "None") == "android.widget.EditText": 

314 view_text = "None" 

315 

316 signature = "[class]%s[resource_id]%s[text]%s[%s,%s,%s]" % ( 

317 DeviceState.__safe_dict_get(view_dict, 'class', "None"), 

318 DeviceState.__safe_dict_get(view_dict, 'resource_id', "None"), 

319 view_text, 

320 DeviceState.__key_if_true(view_dict, 'enabled'), 

321 DeviceState.__key_if_true(view_dict, 'checked'), 

322 DeviceState.__key_if_true(view_dict, 'selected'), 

323 ) 

324 view_dict['signature'] = signature 

325 return signature 

326 

327 @staticmethod 

328 def __get_content_free_view_signature(view_dict): 

329 """ 

330 get the content-free signature of the given view 

331 @param view_dict: dict, an element of list DeviceState.views 

332 @return: 

333 """ 

334 if 'content_free_signature' in view_dict: 

335 return view_dict['content_free_signature'] 

336 content_free_signature = "[class]%s[resource_id]%s" % ( 

337 DeviceState.__safe_dict_get(view_dict, 'class', "None"), 

338 DeviceState.__safe_dict_get(view_dict, 'resource_id', "None"), 

339 ) 

340 view_dict['content_free_signature'] = content_free_signature 

341 return content_free_signature 

342 

343 def __get_view_str(self, view_dict): 

344 """ 

345 get a string which can represent the given view 

346 @param view_dict: dict, an element of list DeviceState.views 

347 @return: 

348 """ 

349 if 'view_str' in view_dict: 

350 return view_dict['view_str'] 

351 view_signature = DeviceState.__get_view_signature(view_dict) 

352 parent_strs = [] 

353 for parent_id in self.get_all_ancestors(view_dict): 

354 parent_strs.append(DeviceState.__get_view_signature(self.views[parent_id])) 

355 parent_strs.reverse() 

356 child_strs = [] 

357 for child_id in self.get_all_children(view_dict): 

358 child_strs.append(DeviceState.__get_view_signature(self.views[child_id])) 

359 child_strs.sort() 

360 view_str = "Activity:%s\nSelf:%s\nParents:%s\nChildren:%s" % ( 

361 self.foreground_activity, 

362 view_signature, 

363 "//".join(parent_strs), 

364 "||".join(child_strs), 

365 ) 

366 import hashlib 

367 

368 view_str = hashlib.md5(view_str.encode('utf-8')).hexdigest() 

369 view_dict['view_str'] = view_str 

370 return view_str 

371 

372 def __get_view_structure(self, view_dict): 

373 """ 

374 get the structure of the given view 

375 :param view_dict: dict, an element of list DeviceState.views 

376 :return: dict, representing the view structure 

377 """ 

378 if 'view_structure' in view_dict: 

379 return view_dict['view_structure'] 

380 width = DeviceState.get_view_width(view_dict) 

381 height = DeviceState.get_view_height(view_dict) 

382 class_name = DeviceState.__safe_dict_get(view_dict, 'class', "None") 

383 children = {} 

384 

385 root_x = view_dict['bounds'][0][0] 

386 root_y = view_dict['bounds'][0][1] 

387 

388 child_view_ids = self.__safe_dict_get(view_dict, 'children') 

389 if child_view_ids: 

390 for child_view_id in child_view_ids: 

391 child_view = self.views[child_view_id] 

392 child_x = child_view['bounds'][0][0] 

393 child_y = child_view['bounds'][0][1] 

394 relative_x, relative_y = child_x - root_x, child_y - root_y 

395 children[ 

396 "(%d,%d)" % (relative_x, relative_y) 

397 ] = self.__get_view_structure(child_view) 

398 

399 view_structure = {"%s(%d*%d)" % (class_name, width, height): children} 

400 view_dict['view_structure'] = view_structure 

401 return view_structure 

402 

403 @staticmethod 

404 def __key_if_true(view_dict, key): 

405 return key if (key in view_dict and view_dict[key]) else "" 

406 

407 @staticmethod 

408 def __safe_dict_get(view_dict, key, default=None): 

409 value = view_dict[key] if key in view_dict else None 

410 return value if value is not None else default 

411 

412 @staticmethod 

413 def get_view_center(view_dict): 

414 """ 

415 return the center point in a view 

416 @param view_dict: dict, an element of DeviceState.views 

417 @return: a pair of int 

418 """ 

419 bounds = view_dict['bounds'] 

420 return (bounds[0][0] + bounds[1][0]) / 2, (bounds[0][1] + bounds[1][1]) / 2 

421 

422 @staticmethod 

423 def get_view_width(view_dict): 

424 """ 

425 return the width of a view 

426 @param view_dict: dict, an element of DeviceState.views 

427 @return: int 

428 """ 

429 bounds = view_dict['bounds'] 

430 return int(math.fabs(bounds[0][0] - bounds[1][0])) 

431 

432 @staticmethod 

433 def get_view_height(view_dict): 

434 """ 

435 return the height of a view 

436 @param view_dict: dict, an element of DeviceState.views 

437 @return: int 

438 """ 

439 bounds = view_dict['bounds'] 

440 return int(math.fabs(bounds[0][1] - bounds[1][1])) 

441 

442 def get_all_ancestors(self, view_dict): 

443 """ 

444 Get temp view ids of the given view's ancestors 

445 :param view_dict: dict, an element of DeviceState.views 

446 :return: list of int, each int is an ancestor node id 

447 """ 

448 result = [] 

449 parent_id = self.__safe_dict_get(view_dict, 'parent', -1) 

450 if 0 <= parent_id < len(self.views): 

451 result.append(parent_id) 

452 result += self.get_all_ancestors(self.views[parent_id]) 

453 return result 

454 

455 def get_all_children(self, view_dict): 

456 """ 

457 Get temp view ids of the given view's children 

458 :param view_dict: dict, an element of DeviceState.views 

459 :return: set of int, each int is a child node id 

460 """ 

461 children = self.__safe_dict_get(view_dict, 'children') 

462 if not children: 

463 return set() 

464 children = set(children) 

465 for child in children: 

466 children_of_child = self.get_all_children(self.views[child]) 

467 children.union(children_of_child) 

468 return children 

469 

470 def get_app_activity_depth(self, app): 

471 """ 

472 Get the depth of the app's activity in the activity stack 

473 :param app: App 

474 :return: the depth of app's activity, -1 for not found 

475 """ 

476 depth = 0 

477 for activity_str in self.activity_stack: 

478 if not activity_str: 

479 return -1 

480 if app.package_name in activity_str: 

481 return depth 

482 depth += 1 

483 return -1 

484 

485 def get_possible_input(self): 

486 """ 

487 Get a list of possible input events for this state 

488 :return: list of InputEvent 

489 """ 

490 if self.possible_events: 

491 return [] + self.possible_events 

492 possible_events = [] 

493 enabled_view_ids = [] 

494 touch_exclude_view_ids = set() 

495 for view_dict in self.views: 

496 # exclude navigation bar if exists 

497 if ( 

498 self.__safe_dict_get(view_dict, 'enabled') 

499 and self.__safe_dict_get(view_dict, 'visible') 

500 and self.__safe_dict_get(view_dict, 'resource_id') 

501 not in [ 

502 'android:id/navigationBarBackground', 

503 'android:id/statusBarBackground', 

504 ] 

505 and self.__safe_dict_get(view_dict, "package") != "com.ohos.sceneboard" 

506 ): 

507 enabled_view_ids.append(view_dict['temp_id']) 

508 

509 

510 enabled_view_ids.reverse() 

511 

512 for view_id in enabled_view_ids: 

513 if self.__safe_dict_get(self.views[view_id], 'clickable') and not ( 

514 '.widget.EditText' in self.__safe_dict_get(self.views[view_id], 'class') 

515 ): 

516 

517 possible_events.append(TouchEvent(view=self.views[view_id])) 

518 touch_exclude_view_ids.add(view_id) 

519 

520 touch_exclude_view_ids = touch_exclude_view_ids.union( 

521 self.get_all_children(self.views[view_id]) 

522 ) 

523 

524 if "org.y20k.transistor" in self.foreground_activity and \ 

525 self.views[view_id]['resource_id'] == "org.y20k.transistor:id/player_sheet": 

526 possible_events.append(ScrollEvent(view=self.views[view_id], direction="UP")) 

527 if "org.y20k.transistor" in self.foreground_activity and \ 

528 self.views[view_id]['resource_id'] == "org.y20k.transistor:id/station_card": 

529 possible_events.append(ScrollEvent(view=self.views[view_id], direction="RIGHT")) 

530 

531 for view_id in enabled_view_ids: 

532 if self.__safe_dict_get(self.views[view_id], 'scrollable'): 

533 possible_events.append( 

534 ScrollEvent(view=self.views[view_id], direction="UP") 

535 ) 

536 possible_events.append( 

537 ScrollEvent(view=self.views[view_id], direction="DOWN") 

538 ) 

539 possible_events.append( 

540 ScrollEvent(view=self.views[view_id], direction="LEFT") 

541 ) 

542 possible_events.append( 

543 ScrollEvent(view=self.views[view_id], direction="RIGHT") 

544 ) 

545 

546 for view_id in enabled_view_ids: 

547 if self.__safe_dict_get(self.views[view_id], 'checkable'): 

548 possible_events.append(TouchEvent(view=self.views[view_id])) 

549 touch_exclude_view_ids.add(view_id) 

550 # fix a bug: add union return values 

551 touch_exclude_view_ids = touch_exclude_view_ids.union( 

552 self.get_all_children(self.views[view_id]) 

553 ) 

554 

555 for view_id in enabled_view_ids: 

556 # add long click event and do not generate the "long click" event for EditText 

557 if self.__safe_dict_get(self.views[view_id], 'long_clickable') and not self.__safe_dict_get(self.views[view_id], 'class') == 'android.widget.EditText': 

558 possible_events.append(LongTouchEvent(view=self.views[view_id])) 

559 

560 for view_id in enabled_view_ids: 

561 if self.__safe_dict_get(self.views[view_id], 'editable'): 

562 from hypothesis import strategies as st 

563 import string 

564 sample_text = st.text(alphabet=string.printable,min_size=0, max_size=8).example() 

565 if random.random() < 0.5: 

566 sample_text = st.text(alphabet=string.ascii_letters,min_size=0, max_size=8).example() 

567 possible_events.append( 

568 SetTextEvent(view=self.views[view_id], text=sample_text) 

569 ) 

570 # add search event for editable view that contains search in its resource id 

571 if self.__safe_dict_get(self.views[view_id], 'resource_id') is not None and "search" in self.__safe_dict_get(self.views[view_id], 'resource_id'): 

572 sample_text = st.text(alphabet=string.printable,min_size=1, max_size=2).example() 

573 if random.random() < 0.5: 

574 sample_text = st.text(alphabet=string.ascii_letters,min_size=1, max_size=2).example() 

575 possible_events.append( 

576 SearchEvent() 

577 ) 

578 possible_events.append( 

579 SetTextAndSearchEvent(text=sample_text) 

580 ) 

581 

582 touch_exclude_view_ids.add(view_id) 

583 # TODO figure out what event can be sent to editable views 

584 pass 

585 # for those views that (1) have not been handled, and (2) are leaf views, generate touch events 

586 for view_id in enabled_view_ids: 

587 if view_id in touch_exclude_view_ids: 

588 continue 

589 children = self.__safe_dict_get(self.views[view_id], 'children') 

590 if children and len(children) > 0: 

591 continue 

592 

593 # fix a possible bug: we still need to check the property 

594 # before we add them into "possible_events" 

595 if self.__safe_dict_get( 

596 self.views[view_id], 'clickable' 

597 ) or self.__safe_dict_get(self.views[view_id], 'checkable'): 

598 possible_events.append(TouchEvent(view=self.views[view_id])) 

599 

600 # For old Android navigation bars 

601 # possible_events.append(KeyEvent(name="MENU")) 

602 

603 self.possible_events = possible_events 

604 return [] + possible_events 

605 

606 

607 def get_text_representation(self, merge_buttons=False): 

608 """ 

609 Get a text representation of current state 

610 """ 

611 enabled_view_ids = [] 

612 for view_dict in self.views: 

613 # exclude navigation bar if exists 

614 if self.__safe_dict_get(view_dict, 'visible') and \ 

615 self.__safe_dict_get(view_dict, 'resource_id') not in \ 

616 ['android:id/navigationBarBackground', 

617 'android:id/statusBarBackground'] and \ 

618 self.__safe_dict_get(view_dict, "package") != "com.ohos.sceneboard": 

619 enabled_view_ids.append(view_dict['temp_id']) 

620 

621 text_frame = "<p id=@ text='&' attr=null bounds=null>#</p>" 

622 btn_frame = "<button id=@ text='&' attr=null bounds=null>#</button>" 

623 checkbox_frame = "<checkbox id=@ text='&' attr=null bounds=null>#</checkbox>" 

624 input_frame = "<input id=@ text='&' attr=null bounds=null>#</input>" 

625 scroll_frame = "<scrollbar id=@ attr=null bounds=null></scrollbar>" 

626 

627 view_descs = [] 

628 indexed_views = [] 

629 # available_actions = [] 

630 removed_view_ids = [] 

631 

632 for view_id in enabled_view_ids: 

633 if view_id in removed_view_ids: 

634 continue 

635 # print(view_id) 

636 view = self.views[view_id] 

637 clickable = self._get_self_ancestors_property(view, 'clickable') 

638 scrollable = self.__safe_dict_get(view, 'scrollable') 

639 checkable = self._get_self_ancestors_property(view, 'checkable') 

640 long_clickable = self._get_self_ancestors_property(view, 'long_clickable') 

641 editable = self.__safe_dict_get(view, 'editable') 

642 actionable = clickable or scrollable or checkable or long_clickable or editable 

643 checked = self.__safe_dict_get(view, 'checked', default=False) 

644 selected = self.__safe_dict_get(view, 'selected', default=False) 

645 content_description = self.__safe_dict_get(view, 'content_description', default='') 

646 view_text = self.__safe_dict_get(view, 'text', default='') 

647 # TODO: how to process the class? 

648 # view_class = self.__safe_dict_get(view, 'class').split('.')[-1] 

649 bounds = self.__safe_dict_get(view, 'bounds') 

650 view_bounds = f'{bounds[0][0]},{bounds[0][1]},{bounds[1][0]},{bounds[1][1]}' 

651 if not content_description and not view_text and not scrollable: # actionable? 

652 continue 

653 

654 # text = self._merge_text(view_text, content_description) 

655 # view_status = '' 

656 view_local_id = str(len(view_descs)) 

657 if editable: 

658 view_desc = input_frame.replace('@', view_local_id).replace('#', view_text) 

659 if content_description: 

660 view_desc = view_desc.replace('&', content_description) 

661 else: 

662 view_desc = view_desc.replace(" text='&'", "") 

663 # available_actions.append(SetTextEvent(view=view, text='HelloWorld')) 

664 elif checkable: 

665 view_desc = checkbox_frame.replace('@', view_local_id).replace('#', view_text) 

666 if content_description: 

667 view_desc = view_desc.replace('&', content_description) 

668 else: 

669 view_desc = view_desc.replace(" text='&'", "") 

670 # available_actions.append(TouchEvent(view=view)) 

671 elif clickable: # or long_clickable 

672 if merge_buttons: 

673 # below is to merge buttons, led to bugs 

674 clickable_ancestor_id = self._get_ancestor_id(view=view, key='clickable') 

675 if not clickable_ancestor_id: 

676 clickable_ancestor_id = self._get_ancestor_id(view=view, key='checkable') 

677 clickable_children_ids = self._extract_all_children(id=clickable_ancestor_id) 

678 if view_id not in clickable_children_ids: 

679 clickable_children_ids.append(view_id) 

680 view_text, content_description = self._merge_text(clickable_children_ids) 

681 checked = self._get_children_checked(clickable_children_ids) 

682 # end of merging buttons 

683 view_desc = btn_frame.replace('@', view_local_id).replace('#', view_text) 

684 if content_description: 

685 view_desc = view_desc.replace('&', content_description) 

686 else: 

687 view_desc = view_desc.replace(" text='&'", "") 

688 # available_actions.append(TouchEvent(view=view)) 

689 if merge_buttons: 

690 for clickable_child in clickable_children_ids: 

691 if clickable_child in enabled_view_ids and clickable_child != view_id: 

692 removed_view_ids.append(clickable_child) 

693 elif scrollable: 

694 # print(view_id, 'continued') 

695 view_desc = scroll_frame.replace('@', view_local_id) 

696 # available_actions.append(ScrollEvent(view=view, direction='DOWN')) 

697 # available_actions.append(ScrollEvent(view=view, direction='UP')) 

698 else: 

699 view_desc = text_frame.replace('@', view_local_id).replace('#', view_text) 

700 if content_description: 

701 view_desc = view_desc.replace('&', content_description) 

702 else: 

703 view_desc = view_desc.replace(" text='&'", "") 

704 # available_actions.append(TouchEvent(view=view)) 

705 

706 allowed_actions = ['touch'] 

707 special_attrs = [] 

708 if editable: 

709 allowed_actions.append('set_text') 

710 if checkable: 

711 allowed_actions.extend(['select', 'unselect']) 

712 allowed_actions.remove('touch') 

713 if scrollable: 

714 allowed_actions.extend(['scroll up', 'scroll down']) 

715 allowed_actions.remove('touch') 

716 if long_clickable: 

717 allowed_actions.append('long_touch') 

718 if checked or selected: 

719 special_attrs.append('selected') 

720 view['allowed_actions'] = allowed_actions 

721 view['special_attrs'] = special_attrs 

722 view['local_id'] = view_local_id 

723 if len(special_attrs) > 0: 

724 special_attrs = ','.join(special_attrs) 

725 view_desc = view_desc.replace("attr=null", f"attr={special_attrs}") 

726 else: 

727 view_desc = view_desc.replace(" attr=null", "") 

728 view_desc = view_desc.replace("bounds=null", f"bound_box={view_bounds}") 

729 view_descs.append(view_desc) 

730 view['desc'] = view_desc.replace(f' id={view_local_id}', '').replace(f' attr={special_attrs}', '')