saui_pr4.latticeui

1 #!/usr/bin/env python 2 3 # Copyright (c) 2007 Carnegie Mellon University. 4 # 5 # You may modify and redistribute this file under the same terms as 6 # the CMU Sphinx system. See 7 # http://cmusphinx.sourceforge.net/html/LICENSE for more information. 8 # 9 # Briefly, don't remove the copyright. Otherwise, do what you like. 10 11 """ 12 Model and view/controller classes for a multitouch ASR error 13 correction implementation. 14 15 This is part of project 4 in 05-631 Software Architecture for User 16 Interfaces, Fall 2007. 17 """ 18 19 __author__ = "David Huggins-Daines <dhuggins@cs.cmu.edu>" 20 21 # System imports 22 import pygtk 23 pygtk.require('2.0') 24 import gtk 25 import pango 26 import sys 27 import math 28 29 # Local imports 30 sys.path.insert(0, ".") 31 import sphinx.lattice 32 import sphinx.arpalm 33

34 -class LatticeWord(object):

35 """Word in a LatticeModel.""" 36 __slots__ = ['sym', 'start', 'end', 'prob', 'base_prob']

37 - def __init__(self, sym, start, end, prob=0):

38 """ 39 Initialize a LatticeWord. 40 41 @param sym: Word string 42 @type sym: string 43 @param start: Start time 44 @type start: int 45 @param end: End time 46 @type end: int 47 @param prob: Posterior log-probability 48 @type prob: float 49 """ 50 self.sym = sym #: Word string 51 self.start = start #: Start time of word 52 self.end = end #: End time of word 53 self.prob = prob #: Posterior log-probability of word 54 self.base_prob = prob #: ASR-derived posterior log-probability

55

56 - def set_prob(self, prob):

57 """ 58 Update the posterior probability of this word 59 60 LatticeWords distinguish between their inherent posterior 61 probability, which is derived purely from ASR results, and 62 their effective posterior probability, which can be modified 63 based on user input. 64 65 @param prob: New posterior log-probability 66 @type prob: float 67 """ 68 self.prob = prob

69

70 - def reset_prob(self):

71 """ 72 Reset the posterior probability to the inherent value 73 """ 74 self.prob = self.base_prob

75

76 -class LatticeCloud(object):

77 """ 78 An expandable "cloud" of hypothesized words. 79 80 A cloud has two dimensions: time and beam width. The cloud 81 contains all words which fall inside the specified start and 82 end points and are within a certain ratio from the best 83 posterior probability. 84 """

85 - def __init__(self, dag, start, end, beam=0.0):

86 """ 87 Create a LatticeCloud object. 88 89 @param dag: Word lattice to initialize this object with. 90 @type dag: sphinx.lattice.Dag 91 @param start: Start time for this cloud. 92 @type start: int 93 @param end: End time for this cloud. 94 @type end: int 95 @param beam: Beam width (log-probability ratio). 96 @type beam: float 97 """ 98 self.dag = dag #: DAG 99 self.set_beam(beam) 100 self.set_time_extents(start, end)

101

102 - def set_beam(self, beam):

103 """ 104 Set the beam width for the cloud. 105 106 @param beam: Beam width (log-probability ratio). 107 @type beam: float 108 """ 109 self.beam = beam #: Beam width of cloud

110

111 - def set_time_extents(self, start, end):

112 """ 113 Set the time extents for the cloud. 114 115 @param start: Start time for this cloud. 116 @type start: int 117 @param end: End time for this cloud. 118 @type end: int 119 """ 120 self.start = start #: Start time of cloud 121 self.end = end #: End time of cloud 122 self.max = -10000 #: maximum posterior probability 123 self.min = 0 #: minimum posterior probability 124 # Scan the DAG to get all nodes in the given range 125 self.nodes = [] #: all nodes in this span 126 for nodes in self.dag[self.start:self.end]: 127 # Look through these nodes and their edges 128 for node in nodes.itervalues(): 129 # Skip filler nodes 130 if sphinx.lattice.is_filler(node.sym) \ 131 or node.sym in ('<s>', '</s>'): 132 continue 133 best_edge = None 134 for ef, ascr in node.exits: 135 # Look for edges fully contained in this region 136 if ef > self.start and ef <= self.end: 137 # Look for the highest and lowest posterior probability 138 ascr, alpha, beta, post = ascr 139 if best_edge == None or post > best_edge[1]: 140 best_edge = (ef, post) 141 if post > self.max: 142 self.max = post 143 if post < self.min: 144 self.min = post 145 if best_edge: 146 self.nodes.append(LatticeWord(node.sym, node.entry, 147 *best_edge))

148

149 - def __iter__(self):

150 """Return an iterator over the words in this cloud.""" 151 for node in self.nodes: 152 if node.prob >= self.max - self.beam: 153 yield node

154

155 -class LatticeModel(object):

156 """ 157 Model of a word lattice obtained from speech recognition, along 158 with a possibly user-specified correct hypothesis. 159 160 It provides alternate words with associated probabilities for 161 visualization, and tracks the user's corrected hypothesis. 162 """

163 - def __init__(self, dag, lm=None, lw=9.5, wip=0.5):

164 """ 165 Create a LatticeModel object. 166 167 @param dag: Word lattice to initialize this object with. 168 @type dag: sphinx.lattice.Dag 169 """ 170 self.lm = self.dag = None 171 self.set_lm(lm, lw, wip) 172 self.set_dag(dag)

173

174 - def set_lm(self, lm=None, lw=9.5, wip=0.5):

175 """ 176 Set the language model and update the best hypothesis and 177 posterior probabilities with the new model. 178 179 @param lm: Language model to use in computation 180 @type lm: sphinx.arpalm.ArpaLM (or equivalent) 181 @param lw: Language model weight 182 @type lw: float 183 @param wip: Word insertion penalty 184 @type wip: float 185 """ 186 self.lm = lm 187 self.lw = lw 188 self.wip = wip 189 # Recompute the best path and posteriors 190 if self.dag: 191 self.set_dag(self.dag, lm, lw, wip)

192

193 - def set_dag(self, dag, lm=None, lw=9.5, wip=0.5):

194 """ 195 Set the DAG and update the best hypothesis from its best path. 196 197 @param dag: DAG to use. 198 @type dag: sphinx.lattice.Dag (or equivalent) 199 @param lm: Language model to use in computation of best path 200 @type lm: sphinx.arpalm.ArpaLM (or equivalent) 201 @param lw: Language model weight 202 @type lw: float 203 @param wip: Word insertion penalty 204 @type wip: float 205 """ 206 # Do bestpath search to get 1-best hypothesis (moving the frame 207 # indices to be the end rather than start frames for each word) 208 bt = dag.backtrace(dag.bestpath(self.lm, self.lw, self.wip)) 209 # Now annotate the lattice with posterior probabilties 210 dag.posterior(self.lm, self.lw, self.wip) 211 self.hyp = [] 212 for i, node in enumerate(bt): 213 prob = None 214 if i == len(bt)-1: 215 end = len(dag) 216 else: 217 end = bt[i+1].entry 218 for frame, score in node.exits: 219 if frame == end: 220 ascr, alpha, beta, prob = score 221 self.hyp.append(LatticeWord(node.sym, node.entry, end, prob)) 222 self.dag = dag

223

224 - def get_hyp(self):

225 """ 226 Retrieve the best hypothesis from a LatticeModel 227 228 @return: The top hypothesis, annotated with start and end 229 frames and posterior probabilities 230 @rtype: list of LatticeWord 231 """ 232 return self.hyp

233

234 - def set_hyp(self, hyp):

235 """ 236 Set the best hypothesis 237 238 @param hyp: The new top hypothesis 239 @type hyp: list of LatticeWord 240 """ 241 self.hyp = hyp

242

243 - def get_hyp_text(self):

244 """ 245 Retrieve the text of the top hypothesis 246 247 @return: The top hypothesis text 248 @rtype: string 249 """ 250 return " ".join([x.sym 251 for x in self.hyp 252 if not (sphinx.lattice.is_filler(x.sym) 253 or x.sym in ('<s>', '</s>'))])

254

255 - def get_cloud(self, word):

256 """ 257 Retrieve a cloud around a hypothesis word. 258 259 @param word: Word in a hypothesis 260 @type word: LatticeWord 261 @return: Cloud around this word with beam width 0 262 @rtype: LatticeCloud 263 """ 264 return LatticeCloud(self.dag, word.start, word.end)

265

266 -class LatticeView(gtk.DrawingArea):

267 """ 268 Handles the visual presentation of word lattices for multitouch 269 error correction. Since user input and display are tightly 270 coupled, there is no separate controller object. 271 272 @ivar model: Model object representing the word lattice 273 @type model: LatticeModel 274 275 @ivar words: Words in the display 276 @type words: set(LatticeView.DisplayWord) 277 278 @ivar clouds: Clouds in the display 279 @type clouds: set(LatticeView.DisplayCloud) 280 281 @ivar drag_state: State of dragging/expansion 282 @type drag_state: int 283 """ 284 XPADDING = 25 #: Horizontal padding to add to the edges of the text 285 YPADDING = 150 #: Vertical expansion area to add to the edges of the text 286 FONTSIZE = 24. #: Basic font size for text 287 PROBSCALE=0.75 #: Squashing factor to apply to probabilities 288 289 NO_DRAG = 0 #: No dragging or expansion currently in progress 290 PRE_DRAG = 2 #: Dragging might start if the mouse moves 291 IN_DRAG = 3 #: Dragging is going on 292

293 - class DisplayWord(object):

294 """ 295 One word to be displayed in the lattice view. 296 297 In addition to a lattice entry, these objects also keep track 298 of their location and size within the lattice view. 299 300 @ivar context: Pango context for drawing this word 301 @type context: pango.Context 302 @ivar layout: Pango layout for drawing this word 303 @type layout: pango.Layout 304 @ivar node: Lattice node corresponding to this word 305 @type node: LatticeWord 306 @ivar x: Horizontal origin (top-left of logical bounds) 307 @type x: int 308 @ivar y: Horizontal origin (top-left of logical bounds) 309 @type y: int 310 @ivar scale: Scaling factor applied when drawing 311 @type scale: float 312 """

313 - def __init__(self, context, desc, node, x, y, scale=1.0):

314 """ 315 Initialize a DisplayWord. 316 317 @param context: Pango context for drawing this word 318 @type context: pango.Context 319 @param desc: Pango font description for drawing this word 320 @type desc: pango.FontDescription 321 @param node: Lattice node corresponding to this word 322 @type node: LatticeWord 323 @param x: Horizontal origin (top-left of logical bounds) 324 @type x: int 325 @param y: Horizontal origin (top-left of logical bounds) 326 @type y: int 327 @param scale: Scaling factor applied when drawing 328 @type scale: float 329 """ 330 self.context = context 331 self.desc = desc 332 self.node = node 333 self.x = x 334 self.y = y 335 self.scale = scale 336 337 # Now create a layout for this word 338 self.layout = pango.Layout(self.context) 339 self.layout.set_text(node.sym + " ") 340 self.layout.set_font_description(desc)

341

342 - def contains(self, x, y):

343 """ 344 Returns True if point (x,y) is inside this word's bounds. 345 346 @param x: Horizontal position of picking point 347 @type x: int 348 @param y: Vertical position of picking point 349 @type y: int 350 @return: True if point (x,y) is inside this word's bounds 351 @rtype: boolean 352 """ 353 bounds, extents = self.layout.get_pixel_extents() 354 # If the bounds are impossibly narrow, add some extra space 355 xfudge = yfudge = 0 356 if bounds[2] < 20: 357 xfudge = (20 - bounds[2]) / 2 358 if bounds[3] < 20: 359 yfudge = (20 - bounds[3]) / 2 360 return (x >= self.x + bounds[0] - xfudge 361 and y >= self.y + bounds[1] - yfudge 362 and x <= self.x + bounds[0] + bounds[2] + xfudge * 2 363 and y <= self.y + bounds[1] + bounds[3] + yfudge * 2)

364

365 - def set_scale(self, scale):

366 """ 367 Set the scaling factor for this word. 368 369 @param scale: Amount to scale this word in the display. 370 @type scale: float 371 """ 372 self.scale = scale

373

374 - def get_scale(self):

375 """ 376 Return the scaling factor for this word. 377 378 @return: Scaling factor for this word. 379 @rtype: float 380 """ 381 return self.scale

382

383 - def get_actual_size(self):

384 """ 385 Return the actual font size for this word. 386 387 @return: Font size for this word, in pango.SCALE units 388 @rtype: float 389 """ 390 return int(self.desc.get_size() * self.scale)

391

392 - def set_pos(self, x, y):

393 """ 394 Set the position for this word. 395 @param x: Horizontal origin (top-left of logical bounds) 396 @type x: int 397 @param y: Horizontal origin (top-left of logical bounds) 398 @type y: int 399 """ 400 self.x = x 401 self.y = y

402

403 - def get_bounds(self):

404 """ 405 Get the ink bounds of this word in the LatticeView 406 coordinate system. 407 408 @return: the ink bounds of this word in the LatticeView 409 coordinate system. 410 @rtype: (int, int, int, int) 411 """ 412 bounds, extents = self.layout.get_pixel_extents() 413 return (self.x + bounds[0], 414 self.y + bounds[1], 415 int(bounds[2] * self.scale), 416 int(bounds[3] * self.scale))

417

418 - def get_extents(self):

419 """ 420 Get the logical extents of this word in the LatticeView 421 coordinate system. 422 @return: the logical extentsof this word in the LatticeView 423 coordinate system. 424 @rtype: (int, int, int, int) 425 """ 426 bounds, extents = self.layout.get_pixel_extents() 427 return (self.x + extents[0], 428 self.y + extents[1], 429 int(extents[2] * self.scale), 430 int(extents[3] * self.scale))

431

432 - def draw(self, gc):

433 """ 434 Draw this word in a given graphics context. 435 436 @param gc: Cairo context (not to be confused with a Pango 437 context) for drawing. 438 @type gc: cairo.Context 439 """ 440 gc.move_to(self.x, self.y) 441 gc.scale(self.scale, self.scale) 442 gc.show_layout(self.layout)

443

444 - class DisplayCloud(LatticeCloud):

445 """ 446 A "cloud" of words to be displayed in the lattice view. 447 448 This object controls the presentation of and user interaction 449 with a C{LatticeCloud}. It lays out the words from a cloud 450 according to their posterior probability and timing, and 451 responds to resizing and click events. 452 453 The user is able to select a set of correct words within a 454 cloud. These words are recorded by this object internally in 455 the C{correct_words} list. 456 457 @ivar context: Pango context for drawing this cloud 458 @type context: pango.Context 459 @ivar desc: Basic Pango font description for drawing this cloud 460 @type desc: pango.FontDescription 461 @ivar words: Set of word objects contained in this cloud. 462 @type words: set(DisplayWord) 463 @ivar hyp_words: Set of original hypothesis word objects in this cloud 464 @type hyp_words: set(DisplayWord) 465 @ivar corrected_words: Set of user-selected words 466 @type corrected_words: set(DisplayWord) 467 @ivar saved_word: Saved word from a previous click, for use in double-click handling 468 @type saved_word: DisplayWord 469 @ivar extents: Logical extents of this cloud 470 @type extents: [int, int, int, int] 471 @ivar min_extents: Minimum extents of this cloud 472 @type min_extents: (int, int, int, int) 473 @ivar max_extents: Maximum extents of this cloud 474 @type max_extents: (int, int, int, int) 475 @ivar edge_resistance: "Gravity zone" around minimum extents to which we snap. 476 @type edge_resistance: int 477 @ivar sign_x: Sign of a delta which "expands" the cloud horizontally 478 @ivar sign_y: Sign of a delta which "expands" the cloud vertically 479 @ivar scale: Scaling factor for drawing 480 @type scale: float 481 @ivar min_font_size: Minimum font size to use, in pango.SCALE units 482 @type min_font_size: int 483 """

484 - def __init__(self, size, context, desc, model, word):

485 """ 486 Initialize a DisplayCloud. 487 488 @param size: Size of the parent object 489 @type size: (int, int) 490 @param context: Pango context for drawing this cloud 491 @type context: pango.Context 492 @param desc: Basic Pango font description for drawing this cloud 493 @type desc: pango.FontDescription 494 @param model: LatticeModel to get this cloud from. 495 @type model: LatticeModel 496 @param word: DisplayWord to use for the initial extents of this cloud. 497 @type word: LatticeView.DisplayWord 498 """ 499 LatticeCloud.__init__(self, model.dag, word.node.start, word.node.end) 500 self.context = context 501 self.desc = desc 502 self.extents = list(word.get_extents()) 503 # Determine minimum and maximum size 504 self.min_extents = word.get_extents() 505 self.max_extents = (LatticeView.XPADDING, LatticeView.XPADDING, 506 size[0] - LatticeView.XPADDING * 2, 507 size[1] - LatticeView.XPADDING * 2) 508 # Map maximum beam width to available space 509 self.beam_step = (self.max - self.min) / self.max_extents[3] 510 # Map time extents to available space 511 self.time_step = float(self.min_extents[2]) / (self.end - self.start) 512 # Initial scaling factor is unity 513 self.scale = 1.0 514 # Add the current word as the initial one (note that it 515 # might not be the MAP hypothesis) 516 word.set_pos(0,0) 517 self.words = set() 518 self.words.add(word) 519 self.hyp_words = set() 520 self.hyp_words.add(word) 521 self.corrected_words = set() 522 self.saved_word = None 523 self.edge_resistance = 10 524 self.min_font_size = 9 * pango.SCALE 525 self.sign_x = self.sign_y = 0

526

527 - def contains(self, x, y):

528 """ 529 Returns True if point (x,y) is inside this cloud's extents. 530 531 @param x: Horizontal position of picking point 532 @type x: int 533 @param y: Vertical position of picking point 534 @type y: int 535 @return: True if point (x,y) is inside this cloud's extents 536 @rtype: boolean 537 """ 538 return (x >= self.extents[0] 539 and y >= self.extents[1] \ 540 and x <= self.extents[0] + self.extents[2] 541 and y <= self.extents[1] + self.extents[3])

542

543 - def y_expand(self, parent):

544 """ 545 Expand the cloud to its maximum vertical size. 546 547 @param parent: Parent widget 548 @type parent: LatticeView 549 """ 550 self.extents[1] = self.max_extents[1] 551 self.extents[3] = self.max_extents[3] 552 # Map vertical expansion to beam expansion 553 self.set_beam((self.extents[3] - self.min_extents[3]) * self.beam_step) 554 self.update_words() 555 # Queue a redraw of our new extents (add a 5px border 556 # to include the drawn border, this won't be necessary 557 # when it is no longer used) 558 parent.queue_draw_area(self.extents[0] - 5, 559 self.extents[1] - 5, 560 self.extents[2] + 10, 561 self.extents[3] + 10)

562

563 - def get_output_words(self):

564 """ 565 Get the set of corrected words. If no corrections have 566 been made, this will return the set of original hypothesis 567 words encompassed by this cloud. 568 569 @return: The set of corrected (or original) words. Note 570 that this is a reference to this set. If you 571 wish to manipulated it, you must copy it! 572 @rtype: set(DisplayWord) 573 """ 574 if len(self.corrected_words) == 0: 575 return self.hyp_words 576 else: 577 return self.corrected_words

578

579 - def collapse(self, parent):

580 """ 581 Collapse the cloud to its minimum size, retaining any user 582 corrections. 583 584 @param parent: Parent widget 585 @type parent: LatticeView 586 """ 587 # Queue a redraw of previous extents 588 parent.queue_draw_area(self.extents[0] - 5, 589 self.extents[1] - 5, 590 self.extents[2] + 10, 591 self.extents[3] + 10) 592 # Collapse extents back to minimum 593 self.extents[:] = self.min_extents[:] 594 # Take output words to be new "hypothesized" and current words 595 new_words = self.get_output_words().copy() 596 self.words.clear() 597 self.words.update(new_words) 598 self.hyp_words.clear() 599 self.hyp_words.update(new_words) 600 # Remove set of corrected words 601 self.corrected_words.clear() 602 # Lay out corrected words horizontally from left to right 603 lr_words = list(new_words) 604 lr_words.sort(lambda x, y: cmp(x.node.start, y.node.start)) 605 x = 0 606 for dw in lr_words: 607 dw.set_scale(1.0) 608 dw.set_pos(x, 0) 609 dw_ext = dw.get_extents() 610 x += dw_ext[2] 611 # Update scaling factor to make them fit 612 xscale = float(self.extents[2]) / x 613 yscale = float(self.extents[3]) / dw_ext[3] 614 self.scale = min(xscale,yscale)

615

616 - def assimilate(self, parent, other):

617 """ 618 Assimilate a neighbouring (we hope) cloud into this one. 619 620 @param parent: Parent widget 621 @type parent: LatticeView 622 @param other: Cloud to be assimilated 623 @type other: LatticeView.DisplayCloud 624 """ 625 def merge_extents(a,b): 626 """ 627 Merge two sets of extents. This is actually just 628 rectangle union and gtk can do that for us, but... 629 """ 630 new_left = min(a[0], b[0]) 631 new_width = (max(a[0] + a[2], b[0] + b[2]) - new_left) 632 new_top = min(a[1], b[1]) 633 new_height = (max(a[1] + a[3], b[1] + b[3]) - new_top) 634 return (new_left, new_top, new_width, new_height)

635 # Increase minimum extents to include other (maximum 636 # extents should be the same for all clouds) 637 self.min_extents = merge_extents(self.min_extents, 638 other.min_extents) 639 # Increase current extents to include other 640 self.extents[:] = merge_extents(self.extents, 641 other.extents) 642 # Extend timepoints to include other 643 self.set_time_extents(min(self.start, other.start), 644 max(self.end, other.end)) 645 # Assimilate any hypothesis and correction words from other 646 self.hyp_words.update(other.hyp_words) 647 self.corrected_words.update(other.corrected_words) 648 # Now update all words 649 self.update_words() 650 # And redraw our new extents 651 parent.queue_draw_area(self.extents[0] - 2, 652 self.extents[1] - 2, 653 self.extents[2] + 4, 654 self.extents[3] + 4)

655

656 - def find_word(self, x, y):

657 """ 658 Find and return the display word in this cloud which contains point 659 (x,y). 660 @param x: Horizontal pixel position (in parent coordinates) 661 @type x: int 662 @param y: Vertical pixel position (in parent coordinates) 663 @type y: int 664 @return: Word in this cloud which contains point (x,y) 665 @rtype: LatticeView.DisplayWord 666 """ 667 # Translate (x,y) to word space coordinates 668 x = (x - self.extents[0]) / self.scale 669 y = (y - self.extents[1]) / self.scale 670 for w in self.words: 671 if w.contains(x, y): 672 return w 673 return None

674

675 - def double_click(self, parent, x, y):

676 """ 677 React to a double click. 678 679 @param parent: Parent widget 680 @type parent: LatticeView 681 @param x: Click point X 682 @type x: int 683 @param y: Click point Y 684 @type y: int 685 """ 686 # Undo removal of a word on double-click 687 if self.saved_word: 688 self.saved_word.node.set_prob(0) 689 self.corrected_words.add(self.saved_word) 690 self.saved_word = None

691

692 - def click(self, parent, x, y):

693 """ 694 React to a discrete mouse click. 695 696 @param parent: Parent widget 697 @type parent: LatticeView 698 @param x: Click point X 699 @type x: int 700 @param y: Click point Y 701 @type y: int 702 """ 703 # Remove any saved word for double clicking 704 self.saved_word = None 705 dw = self.find_word(x,y) 706 if dw: 707 # Toggle dw in corrected words 708 if dw in self.corrected_words: 709 # Save this so we can restore it if this turns out 710 # to be a double-click 711 self.saved_word = dw 712 self.corrected_words.remove(dw) 713 dw.node.reset_prob() 714 else: 715 # If the user thinks this word is correct, we'll give 716 # it a posterior of 1.0. Actually we should look at 717 # this as a form of Bayesian update, but doing this 718 # ensures that the darn thing will stick around! 719 dw.node.set_prob(0) 720 self.corrected_words.add(dw) 721 # Translate word extents to parent coordinates 722 dw_ext = [int(x * self.scale) for x in dw.get_extents()] 723 dw_ext[0] += self.extents[0] 724 dw_ext[1] += self.extents[1] 725 # Queue a redraw inside those extents 726 parent.queue_draw_area(*dw_ext)

727

728 - def drag_delta(self, parent, x, y, delta_x, delta_y):

729 """ 730 React to drag motion of (delta_x, delta_y) to point (x,y). 731 732 @param parent: Parent widget 733 @type parent: LatticeView 734 @param x: Destination point X 735 @type x: int 736 @param y: Destination point Y 737 @type y: int 738 @param delta_x: Horizontal distance travelled 739 @type delta_x: int 740 @param delta_y: Vertical distance travelled 741 @type delta_y: int 742 @return: The effective (x,y) delta after compensating for 743 drag direction. This indicates the desired 744 change in size for this cloud. 745 @rtype: (int, int) 746 """ 747 # When starting the drag, determine which direction is 748 # "out". We will preserve this for the entire drag, 749 # otherwise it becomes very difficult to fully "compress" 750 # the cloud. 751 if self.sign_y == 0: 752 # If the cloud is collapsed, then clearly whichever 753 # direction the user wanted to go is "expansion" 754 if self.extents[1] == self.min_extents[1]: 755 self.sign_y = delta_y 756 # Otherwise, this depends on whether we're above or 757 # below the centerline. 758 else: 759 center_y = self.extents[1] + self.extents[3] / 2 760 self.sign_y = y - center_y 761 if self.sign_x == 0: 762 # Likewise for sign_x 763 if self.extents[0] == self.min_extents[0]: 764 self.sign_x = delta_x 765 else: 766 center_x = self.extents[0] + self.extents[2] / 2 767 self.sign_x = x - center_x 768 # Now modify the sign of delta in accordance 769 if self.sign_y < 0: delta_y = -delta_y 770 if self.sign_x < 0: delta_x = -delta_x 771 772 # Implementation of edge resistance: 773 # On an "outbound" drag (delta > 0), avoid redrawing until 774 # the extents reach min_extents + edge_resistance 775 if delta_y > 0: 776 resistance_is_futile = (self.min_extents[1] - 777 (self.extents[1] - int(delta_y)) 778 > self.edge_resistance) 779 else: 780 resistance_is_futile = True 781 # On an "inbound" drag (delta < 0), snap immediately to 782 # min_extents once we get within range. 783 if delta_y < 0: 784 you_will_be_assimilated = (self.min_extents[1] - 785 (self.extents[1] - int(delta_y)) 786 < self.edge_resistance) 787 else: 788 you_will_be_assimilated = False 789 # Queue a redraw of our current extents (add a 2px border 790 # to include the outside of the drawn border) 791 if resistance_is_futile: 792 parent.queue_draw_area(self.extents[0] - 2, 793 self.extents[1] - 2, 794 self.extents[2] + 4, 795 self.extents[3] + 4) 796 # Vertical expansion 797 self.extents[1] -= int(delta_y) 798 self.extents[3] += int(delta_y * 2) 799 # Ensure everything is symmetrical and inside bounds 800 if self.extents[1] < self.max_extents[1]: 801 self.extents[3] -= (self.max_extents[1] - self.extents[1]) * 2 802 self.extents[1] = self.max_extents[1] 803 # Don't shrink below min_extents 804 self.extents[1] = min(self.extents[1], self.min_extents[1]) 805 self.extents[3] = max(self.extents[3], self.min_extents[3]) 806 # Snap to minimum size on an inbound drag 807 if you_will_be_assimilated: 808 self.extents[:] = self.min_extents[:] 809 # Redraw unless there is edge resistance 810 if resistance_is_futile: 811 # Map vertical size to beam (don't use delta because we 812 # clamped our size...) 813 self.set_beam((self.extents[3] - self.min_extents[3]) * self.beam_step) 814 # Update words 815 self.update_words() 816 # Queue a redraw of our new extents 817 parent.queue_draw_area(self.extents[0] - 2, 818 self.extents[1] - 2, 819 self.extents[2] + 4, 820 self.extents[3] + 4) 821 return delta_x, delta_y

822

823 - def is_minimum(self):

824 """ 825 Returns True if this cloud is at its minimum size. 826 827 @return: True if this cloud is at its minimum size (duh) 828 @rtype: boolean 829 """ 830 return (self.extents[0] == self.min_extents[0] 831 and self.extents[1] == self.min_extents[1])

832

833 - def end_drag(self, parent, x, y):

834 """ 835 React to ending a drag at point (x,y) 836 837 @param parent: Parent widget 838 @type parent: LatticeView 839 @param x: Destination point X 840 @type x: int 841 @param y: Destination point Y 842 @type y: int 843 """ 844 # Reset the drag direction 845 self.sign_x = self.sign_y = 0

846

847 - def update_words(self):

848 """ 849 Update the list of words to match extents. 850 """ 851 # Obtain the list of words within the current beam 852 words = list(self) 853 if len(words) == 0: 854 # Uh oh, no words. Use the output hypothesis 855 self.words.clear() 856 self.words.update(self.get_output_words()) 857 return 858 # Sort them in reverse by *inherent* probability - this 859 # allows corrected words to retain their original place 860 words.sort(lambda x,y: cmp(y.base_prob, x.base_prob)) 861 # Compute total and maximum probability 862 tprob = 0 863 maxprob = 0 864 for w in words: 865 # Exponentiate to "flatten" probabilities 866 # Also use inherent probability for scaling, so that 867 # they don't jump in size. 868 prob = math.exp(w.base_prob * LatticeView.PROBSCALE) 869 if (prob > maxprob): 870 maxprob = prob 871 tprob += prob 872 maxprob /= tprob 873 # Now generate a list of display words (re-using existing 874 # ones) and lay them out inside the given extents. 875 # Create a mapping of LatticeWord => DisplayWord 876 dw_map = dict([(x.node, x) for x in self.words]) 877 display_words = [] 878 width = height = 0 879 # Keep track of output words which do not 880 for w in words: 881 # Place them horizontally according to time 882 left = int(self.time_step * (w.start - self.start)) 883 if left < 0: left = 0 884 right = int(self.time_step * (w.end - self.start)) 885 if right > width: width = right 886 wprob = math.exp(w.base_prob * LatticeView.PROBSCALE) / tprob 887 # Look for an existing match 888 if w in dw_map: 889 dw = dw_map[w] 890 dw.set_pos(left, height) 891 # max = 1.0, others smaller 892 dw.set_scale(1.0 + wprob - maxprob) 893 else: 894 dw = LatticeView.DisplayWord(self.context, 895 self.desc, w, 896 left, height, 897 # max = 1.0, others smaller 898 1.0 + wprob - maxprob) 899 dw_ext = dw.get_extents() 900 # Center them horizontally within their time slot 901 left += (right - left - dw_ext[2]) / 2 902 if left < 0: left = 0 903 if left + dw_ext[2] > width: width = left + dw_ext[2] 904 dw.set_pos(left, dw_ext[1]) 905 # Increment height 906 height += dw_ext[3] 907 display_words.append(dw) 908 # Find the appropriate scaling factor to fit things 909 xscale = float(self.extents[2]) / width 910 yscale = float(self.extents[3]) / height 911 # Pop any words off the list that are just too small, 912 # readjusting scale as we go. Since they are sorted in 913 # descending order this is a nice linear operation 914 minsize = display_words[-1].get_actual_size() * min(xscale, yscale) 915 # Keep track of corrected words that would otherwise get ditched 916 extra_words = [] 917 while minsize < self.min_font_size \ 918 and len(display_words) > 1: # make sure there is at least one! 919 pinkie = display_words.pop() 920 if pinkie in self.corrected_words: 921 # FIXME: Also maintain minimum size for corrected words 922 extra_words.append(pinkie) 923 else: 924 height -= pinkie.get_extents()[3] 925 yscale = float(self.extents[3]) / height 926 minsize = display_words[-1].get_actual_size() * min(xscale, yscale) 927 # Put the corrected words back 928 display_words.extend(extra_words) 929 # "Reflect" the display words around the center of the 930 # list, such that they become sorted in top-to-bottom 931 # order with the highest probability in the middle. Do 932 # this by removing every other item and prepending these 933 # in reverse order. 934 botwords = display_words[1::2] 935 display_words = display_words[0::2] 936 display_words.reverse() 937 display_words.extend(botwords) 938 # Pad everything out appropriately 939 if xscale < yscale: 940 self.scale = xscale 941 yspace = (self.extents[3] / xscale - height) / len(display_words) 942 xspace = 0 943 else: 944 self.scale = yscale 945 xspace = self.extents[2] / yscale - width 946 yspace = 0 947 height = yspace / 2 948 # Maintain list of extents on the current line, in order 949 # to reduce vertical scaling. 950 line_height = 0 951 line_ext = [] 952 for dw in display_words: 953 dw_ext = dw.get_extents() 954 # Look through extents on current line, if any, to 955 # determine if we need to move down 956 for ext in line_ext: 957 if (dw_ext[0] + dw_ext[2] > ext[0] 958 and dw_ext[0] <= ext[0] + ext[2]): 959 height += line_height + yspace 960 line_ext = [] 961 line_height = 0 962 break 963 dw.set_pos(int(xspace / 2 + dw_ext[0]), int(height)) 964 if int(xspace / 2 + dw_ext[0] + dw_ext[2]) > width: 965 width = int(xspace / 2 + dw_ext[0] + dw_ext[2]) 966 prev_ext = dw_ext 967 # Update line_height and line_ext 968 if dw_ext[3] > line_height: line_height = dw_ext[3] 969 line_ext.append(dw_ext) 970 if line_height: 971 height += line_height + yspace / 2 972 # Adjust yscale one more time to compensate for overlaps 973 xscale = float(self.extents[2]) / width 974 yscale = float(self.extents[3]) / height 975 self.scale = min(xscale, yscale) 976 # And the set of active words 977 self.words = set(display_words)

978

979 - def draw(self, gc):

980 """ 981 Draw this cloud in a given graphics context. 982 983 @param gc: Cairo context (not to be confused with a Pango 984 context) for drawing. 985 @type gc: cairo.Context 986 """ 987 # Stroke a red rectangle around the cloud 988 gc.set_source_rgb(1.0,0,0) 989 gc.rectangle(tuple(self.extents)) 990 gc.stroke() 991 # Translate to set text 992 gc.translate(self.extents[0], self.extents[1]) 993 # Scale to appropriate size 994 gc.scale(self.scale, self.scale) 995 # Draw all words 996 for w in self.words: 997 gc.save() 998 # Draw corrected words in red, others in black 999 if w in self.corrected_words: 1000 gc.set_source_rgb(1.0,0,0) 1001 else: 1002 gc.set_source_rgb(0,0,0) 1003 w.draw(gc) 1004 gc.restore()

1005

1006 - def __init__(self, model):

1007 """ 1008 Construct a LatticeView. 1009 1010 @param model: model containing the word lattice and best 1011 hypothesis to be displayed. 1012 @type model: LatticeModel 1013 """ 1014 # Initialize the base class 1015 gtk.DrawingArea.__init__(self) 1016 self.model = model 1017 # Set up user interaction events 1018 self.add_events(gtk.gdk.BUTTON_PRESS_MASK 1019 | gtk.gdk.BUTTON_RELEASE_MASK 1020 | gtk.gdk.POINTER_MOTION_MASK) 1021 self.connect("button_press_event", self.button_press) 1022 self.connect("button_release_event", self.button_release) 1023 self.connect("motion_notify_event", self.motion_notify) 1024 self.drag_state = self.NO_DRAG #: Current state of dragging 1025 self.drag_word = None #: Word currently being dragged 1026 self.drag_cloud = None #: Cloud currently being dragged 1027 # Set up a handler for ConfigureEvents (resizing, etc) 1028 self.connect("configure_event", self.configure) 1029 # Set up our drawing function which responds to ExposeEvents (i.e. damage) 1030 self.connect("expose_event", self.expose) 1031 # No clouds to start with 1032 self.clouds = set() 1033 # Create the set of individual words to draw 1034 self.words = set() 1035 # Set up the font used for drawing 1036 self.desc = pango.FontDescription() 1037 self.desc.set_size(int(self.FONTSIZE * 1024)) 1038 # Pango draws from the top-left corner 1039 x = self.XPADDING 1040 y = self.YPADDING 1041 for w in self.model.get_hyp(): 1042 if sphinx.lattice.is_filler(w.sym) or w.sym in ('<s>', '</s>'): 1043 continue 1044 dw = self.DisplayWord(self.get_pango_context(), 1045 self.desc.copy_static(), 1046 w, x, y) 1047 self.words.add(dw) 1048 # Advance the reference point by the logical width 1049 x += dw.get_extents()[2] 1050 # Pad out the right side 1051 width = x + self.XPADDING 1052 # And the bottom side (bottom edge is the same for all 1053 # words since it is calculated based on the ascent, descent, 1054 # and leading of the font rather than the actual glyphs) 1055 height = self.YPADDING + dw.get_extents()[3] + self.YPADDING 1056 self.set_size_request(width, height)

1057

1058 - def find_word(self, x, y):

1059 """ 1060 Find and return the display word (if any) which contains point 1061 (x,y). 1062 @param x: Horizontal pixel position inside this view. 1063 @type x: int 1064 @param y: Vertical pixel position inside this view. 1065 @type y: int 1066 @return: Word which contains point (x,y) 1067 @rtype: LatticeView.DisplayWord 1068 """ 1069 for w in self.words: 1070 if w.contains(x, y): 1071 return w 1072 return None

1073

1074 - def find_cloud(self, x, y):

1075 """ 1076 Find and return the display cloud (if any) which contains point 1077 (x,y). 1078 @param x: Horizontal pixel position inside this view. 1079 @type x: int 1080 @param y: Vertical pixel position inside this view. 1081 @type y: int 1082 @return: Cloud which contains point (x,y) 1083 @rtype: LatticeView.DisplayCloud 1084 """ 1085 for c in self.clouds: 1086 if c.contains(x, y): 1087 return c 1088 return None

1089

1090 - def absorb_cloud(self, cloud):

1091 """ 1092 Reclaim output words from a cloud and dispose of it. 1093 1094 @param cloud: The cloud to get rid of 1095 @type cloud: LatticeView.DisplayCloud 1096 """ 1097 cloud.collapse(self) 1098 # Now extract its output words and kill it 1099 for dw in cloud.get_output_words(): 1100 # Translate word extents to parent coordinates 1101 dw_ext = [int(x * cloud.scale) for x in dw.get_extents()] 1102 dw_ext[0] += cloud.extents[0] 1103 dw_ext[1] += cloud.extents[1] 1104 dw.set_pos(dw_ext[0], dw_ext[1]) 1105 dw.set_scale(cloud.scale) 1106 self.words.add(dw) 1107 self.clouds.remove(cloud)

1108

1109 - def extend_cloud(self, cloud, x, y):

1110 """ 1111 Merge cloud with adjoining cloud or word at (x,y). 1112 1113 @param cloud: The cloud to expand 1114 @type cloud: LatticeView.DisplayCloud 1115 @param x: Horizontal pixel position inside adjoining cloud/word 1116 @type x: int 1117 @param y: Vertical pixel position inside adjoining cloud/word 1118 @type y: int 1119 """ 1120 # Figure out what cloud or word we just entered 1121 w = self.find_word(self.drag_x, cloud.extents[1] + cloud.extents[3] / 2) 1122 if w: 1123 # Create a new cloud for this word 1124 new_cloud = self.DisplayCloud(self.get_size_request(), 1125 self.get_pango_context(), 1126 self.desc.copy_static(), 1127 self.model, w) 1128 # Merge it into the current drag cloud 1129 cloud.assimilate(self, new_cloud) 1130 self.words.remove(w) 1131 else: 1132 # Look for a cloud 1133 c = self.find_cloud(self.drag_x, cloud.extents[1] + cloud.extents[3] / 2) 1134 if c: 1135 # Merge it and remove it from our list of clouds 1136 cloud.assimilate(self, c) 1137 self.clouds.remove(c)

1138

1139 - def button_press(self, widget, event):

1140 """ 1141 Repond to button press events. 1142 1143 @param widget: reference to widget where event occurred 1144 @type widget: gtk.Widget 1145 @param event: reference to the event 1146 @type event: gtk.gdk.Event 1147 """ 1148 # Handle doubleclicks 1149 if event.type == gtk.gdk._2BUTTON_PRESS: 1150 # Cancel any pending drags 1151 self.drag_state = self.NO_DRAG 1152 # If this happened in one of our words, it will be self.drag_word 1153 if self.drag_word: 1154 # Turn this word into a maximal cloud 1155 cloud = self.DisplayCloud(self.get_size_request(), 1156 self.get_pango_context(), 1157 self.desc.copy_static(), 1158 self.model, self.drag_word) 1159 # Expand it maximally 1160 cloud.y_expand(self) 1161 self.clouds.add(cloud) 1162 self.words.remove(self.drag_word) 1163 # If this happened in a cloud, it will be self.drag_cloud. 1164 elif self.drag_cloud: 1165 # Pass on the double click to the cloud - this might 1166 # undo a previous click. 1167 self.drag_cloud.double_click(self, event.x, event.y) 1168 # Collapse it, extract its output words, and kill it 1169 self.absorb_cloud(self.drag_cloud) 1170 # Clear any drag word/cloud 1171 self.drag_word = None 1172 self.drag_cloud = None 1173 # Handle button press 1174 elif event.type == gtk.gdk.BUTTON_PRESS: 1175 # Record click location to prepare for possible dragging 1176 self.drag_x = event.x 1177 self.drag_y = event.y 1178 self.drag_time = event.time 1179 # Check to see if this is in one of our words 1180 w = self.find_word(event.x, event.y) 1181 # If so... 1182 if w: 1183 # This might be a drag, but it might not. Prepare for 1184 # that possibility by entering PRE_DRAG state 1185 self.drag_state = self.PRE_DRAG 1186 self.drag_word = w 1187 else: 1188 # Look for a cloud 1189 c = self.find_cloud(event.x, event.y) 1190 if c: 1191 # Enter PRE_DRAG state with this cloud 1192 self.drag_state = self.PRE_DRAG 1193 self.drag_cloud = c 1194 return False

1195

1196 - def button_release(self, widget, event):

1197 """ 1198 Repond to button release events. 1199 1200 @param widget: reference to widget where event occurred 1201 @type widget: gtk.Widget 1202 @param event: reference to the event 1203 @type event: gtk.gdk.Event 1204 """ 1205 # Handle button release (probably won't get any other events 1206 # here actually) 1207 if event.type == gtk.gdk.BUTTON_RELEASE: 1208 # Handle the end of a drag 1209 if self.drag_state == self.IN_DRAG: 1210 # If we had a cloud, then terminate the drag on it 1211 if self.drag_cloud: 1212 self.drag_cloud.end_drag(self, event.x, event.y) 1213 # If it has reached minimum size, then eliminate it 1214 if self.drag_cloud.is_minimum(): 1215 # Collapse it, extract its output words, and kill it 1216 self.absorb_cloud(self.drag_cloud) 1217 self.drag_state = self.NO_DRAG 1218 self.drag_cloud = None 1219 # Otherwise cancel any drag in progress and handle a 1220 # single button press. 1221 elif self.drag_state == self.PRE_DRAG: 1222 self.drag_state = self.NO_DRAG 1223 # If this happened in a cloud (which would be 1224 # self.drag_cloud), then pass it on 1225 if self.drag_cloud: 1226 self.drag_cloud.click(self, event.x, event.y) 1227 self.drag_cloud = None 1228 return False

1229

1230 - def motion_notify(self, widget, event):

1231 """ 1232 Repond to motion notify events. 1233 1234 @param widget: reference to widget where event occurred 1235 @type widget: gtk.Widget 1236 @param event: reference to the event 1237 @type event: gtk.gdk.Event 1238 """ 1239 if event.type == gtk.gdk.MOTION_NOTIFY: 1240 # If we were possibly going to drag, then confirm that we 1241 # are actually dragging by entering IN_DRAG state 1242 # immediately 1243 if self.drag_state == self.PRE_DRAG: 1244 self.drag_state = self.IN_DRAG 1245 # Turn the drag word into a cloud, if needed 1246 if self.drag_word: 1247 self.drag_cloud = self.DisplayCloud(self.get_size_request(), 1248 self.get_pango_context(), 1249 self.desc.copy_static(), 1250 self.model, self.drag_word) 1251 self.clouds.add(self.drag_cloud) 1252 # We no longer have a drag word 1253 self.words.remove(self.drag_word) 1254 self.drag_word = None 1255 # Now fall through to handle dragging 1256 if self.drag_state == self.IN_DRAG: 1257 # Find the difference from the previous drag point 1258 delta_x = event.x - self.drag_x 1259 delta_y = event.y - self.drag_y 1260 delta_time = event.time - self.drag_time 1261 # Update our drag point 1262 self.drag_x = event.x 1263 self.drag_y = event.y 1264 self.drag_time = event.time 1265 # Hand things off to the cloud for expansion (this 1266 # actually only does the Y direction) 1267 self.drag_cloud.drag_delta(self, 1268 self.drag_x, self.drag_y, 1269 delta_x, delta_y) 1270 # X direction works rather differently and needs to be 1271 # handled in the parent. Basically what happens is 1272 # that on outbound drags, once drag_x crosses the 1273 # extent of this cloud plus some heuristically 1274 # determined threshold, this cloud will be merged with 1275 # the neighbouring one. 1276 if (delta_x > 0 1277 and self.drag_cloud.sign_x > 0 1278 and (self.drag_x 1279 > (self.drag_cloud.extents[0] 1280 + self.drag_cloud.extents[2] + 10))): 1281 self.extend_cloud(self.drag_cloud, self.drag_x, self.drag_y) 1282 if (delta_x < 0 1283 and self.drag_cloud.sign_x < 0 1284 and (self.drag_x 1285 < (self.drag_cloud.extents[0] - 10))): 1286 self.extend_cloud(self.drag_cloud, self.drag_x, self.drag_y) 1287 return False

1288

1289 - def configure(self, widget, event):

1290 """ 1291 Respond to configure events. 1292 1293 @param widget: reference to widget where event occurred 1294 @type widget: gtk.Widget 1295 @param event: reference to the event 1296 @type event: gtk.gdk.Event 1297 """ 1298 return False

1299

1300 - def expose(self, widget, event):

1301 """ 1302 Respond to expose events. 1303 1304 @param widget: reference to widget where event occurred 1305 @type widget: gtk.Widget 1306 @param event: reference to the event 1307 @type event: gtk.gdk.Event 1308 """ 1309 # Get a Cairo drawing context 1310 self.context = widget.window.cairo_create() 1311 # Add a rectangle to the current path (ahh.... this is like PostScript) 1312 self.context.rectangle(event.area.x, event.area.y, 1313 event.area.width, event.area.height) 1314 # Clip based on the current path 1315 self.context.clip() 1316 # Now do some drawing 1317 self.draw() 1318 return False

1319

1320 - def draw(self):

1321 """ 1322 Draw or re-draw the lattice presentation 1323 """ 1324 # Get current size of this widget 1325 rect = self.get_allocation() 1326 # Draw a white background 1327 self.context.rectangle(rect) 1328 self.context.set_source_rgb(1,1,1) 1329 self.context.fill() 1330 # Draw words and clouds 1331 for dw in self.words: 1332 self.context.save() 1333 # Draw words in black 1334 self.context.set_source_rgb(0,0,0) 1335 dw.draw(self.context) 1336 self.context.restore() 1337 for dc in self.clouds: 1338 self.context.save() 1339 dc.draw(self.context) 1340 self.context.restore()

1341

1342 -def main():

1343 """ 1344 Simple test program for saui.latticeui.LatticeView. 1345 """ 1346 # Create the toplevel window 1347 window = gtk.Window() 1348 # Make sure we will exit when the window is closed 1349 window.connect("delete-event", gtk.main_quit) 1350 # Load a test lattice 1351 dag = sphinx.lattice.Dag("lattice/test8.lat.gz") 1352 # Load the language model 1353 lm = None #sphinx.arpalm.ArpaLM("model/cleanmail.arpa") 1354 # Create the LatticeModel 1355 model = LatticeModel(dag, lm, 9.5, 0.5) 1356 # Create the LatticeView widget 1357 lattice = LatticeView(model) 1358 window.add(lattice) 1359 # Pack everything in the window 1360 window.show_all() 1361 # Run the main loop 1362 gtk.main()

1363 1364 if __name__ == '__main__': 1365 main() 1366

Source Code for Module saui_pr4.latticeui