Skip to content

Commit 835f537

Browse files
coding-chimpcoding-chimp
authored
Improve performance for tables with composite primary keys (#1686)
1 parent 251b08d commit 835f537

2 files changed

Lines changed: 673 additions & 49 deletions

File tree

‎go/sql/builder.go‎

Lines changed: 314 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -281,23 +281,34 @@ func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName strin
281281
sharedColumnsListing := strings.Join(sharedColumns, ", ")
282282

283283
uniqueKey = EscapeName(uniqueKey)
284+
transactionalClause := ""
285+
if transactionalTable {
286+
if noWait {
287+
transactionalClause = "for share nowait"
288+
} else {
289+
transactionalClause = "lock in share mode"
290+
}
291+
}
284292
var minRangeComparisonSign = GreaterThanComparisonSign
285293
if includeRangeStartValues {
286294
minRangeComparisonSign = GreaterThanOrEqualsComparisonSign
287295
}
296+
297+
if uniqueKeyColumns.Len() == 2 {
298+
return buildRangeInsertQueryTwoColumn(
299+
databaseName, originalTableName, ghostTableName,
300+
sharedColumnsListing, mappedSharedColumnsListing,
301+
uniqueKey, uniqueKeyColumns,
302+
rangeStartValues, rangeEndValues,
303+
rangeStartArgs, rangeEndArgs,
304+
minRangeComparisonSign, transactionalClause,
305+
)
306+
}
288307
rangeStartComparison, rangeExplodedArgs, err := BuildRangeComparison(uniqueKeyColumns.Names(), rangeStartValues, rangeStartArgs, minRangeComparisonSign)
289308
if err != nil {
290309
return "", explodedArgs, err
291310
}
292311
explodedArgs = append(explodedArgs, rangeExplodedArgs...)
293-
transactionalClause := ""
294-
if transactionalTable {
295-
if noWait {
296-
transactionalClause = "for share nowait"
297-
} else {
298-
transactionalClause = "lock in share mode"
299-
}
300-
}
301312
rangeEndComparison, rangeExplodedArgs, err := BuildRangeComparison(uniqueKeyColumns.Names(), rangeEndValues, rangeEndArgs, LessThanOrEqualsComparisonSign)
302313
if err != nil {
303314
return "", explodedArgs, err
@@ -323,6 +334,91 @@ func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName strin
323334
return result, explodedArgs, nil
324335
}
325336

337+
func sameFirstColumnValue(rangeStartArgs, rangeEndArgs []interface{}) bool {
338+
if len(rangeStartArgs) == 0 || len(rangeEndArgs) == 0 {
339+
return false
340+
}
341+
return fmt.Sprintf("%v", rangeStartArgs[0]) == fmt.Sprintf("%v", rangeEndArgs[0])
342+
}
343+
344+
func buildRangeInsertQueryTwoColumn(
345+
databaseName, originalTableName, ghostTableName string,
346+
sharedColumnsListing, mappedSharedColumnsListing string,
347+
uniqueKey string,
348+
uniqueKeyColumns *ColumnList,
349+
rangeStartValues, rangeEndValues []string,
350+
rangeStartArgs, rangeEndArgs []interface{},
351+
minRangeComparisonSign ValueComparisonSign,
352+
transactionalClause string,
353+
) (result string, explodedArgs []interface{}, err error) {
354+
cols := uniqueKeyColumns.Columns()
355+
356+
if len(cols) != len(rangeStartValues) {
357+
return "", explodedArgs, fmt.Errorf("got %d columns but %d rangeStartValues in buildRangeInsertQueryTwoColumn", len(cols), len(rangeStartValues))
358+
}
359+
if len(cols) != len(rangeEndValues) {
360+
return "", explodedArgs, fmt.Errorf("got %d columns but %d rangeEndValues in buildRangeInsertQueryTwoColumn", len(cols), len(rangeEndValues))
361+
}
362+
if len(cols) != len(rangeStartArgs) {
363+
return "", explodedArgs, fmt.Errorf("got %d columns but %d rangeStartArgs in buildRangeInsertQueryTwoColumn", len(cols), len(rangeStartArgs))
364+
}
365+
if len(cols) != len(rangeEndArgs) {
366+
return "", explodedArgs, fmt.Errorf("got %d columns but %d rangeEndArgs in buildRangeInsertQueryTwoColumn", len(cols), len(rangeEndArgs))
367+
}
368+
369+
col1Name := EscapeName(cols[0].Name)
370+
col2Name := EscapeName(cols[1].Name)
371+
col1StartVal := rangeStartValues[0]
372+
col2StartVal := rangeStartValues[1]
373+
col1EndVal := rangeEndValues[0]
374+
col2EndVal := rangeEndValues[1]
375+
col2StartOp := string(minRangeComparisonSign)
376+
fromClause := fmt.Sprintf("%s.%s force index (%s)", databaseName, originalTableName, uniqueKey)
377+
378+
if sameFirstColumnValue(rangeStartArgs, rangeEndArgs) {
379+
result = fmt.Sprintf(`
380+
insert /* gh-ost %s.%s */ ignore
381+
into
382+
%s.%s
383+
(%s)
384+
(
385+
select %s
386+
from
387+
%s.%s
388+
force index (%s)
389+
where (%s = %s and %s %s %s and %s <= %s)
390+
%s
391+
)`,
392+
databaseName, originalTableName,
393+
databaseName, ghostTableName, mappedSharedColumnsListing,
394+
sharedColumnsListing,
395+
databaseName, originalTableName, uniqueKey,
396+
col1Name, col1StartVal, col2Name, col2StartOp, col2StartVal, col2Name, col2EndVal,
397+
transactionalClause,
398+
)
399+
explodedArgs = append(explodedArgs, rangeStartArgs[0], rangeStartArgs[1], rangeEndArgs[1])
400+
return result, explodedArgs, nil
401+
}
402+
403+
part1, part2, part3, explodedArgs := buildTwoColumnUnionParts(
404+
sharedColumnsListing, fromClause,
405+
col1Name, col2Name,
406+
col1StartVal, col2StartVal, col1EndVal, col2EndVal,
407+
col2StartOp, transactionalClause,
408+
rangeStartArgs, rangeEndArgs,
409+
)
410+
411+
result = fmt.Sprintf(`
412+
insert /* gh-ost %s.%s */ ignore
413+
into %s.%s (%s)
414+
%s union all %s union all %s`,
415+
databaseName, originalTableName,
416+
databaseName, ghostTableName, mappedSharedColumnsListing,
417+
part1, part2, part3,
418+
)
419+
return result, explodedArgs, nil
420+
}
421+
326422
func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, mappedSharedColumns []string, uniqueKey string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, includeRangeStartValues bool, transactionalTable bool, noWait bool) (result string, explodedArgs []interface{}, err error) {
327423
rangeStartValues := buildColumnsPreparedValues(uniqueKeyColumns)
328424
rangeEndValues := buildColumnsPreparedValues(uniqueKeyColumns)
@@ -340,6 +436,11 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string
340436
if includeRangeStartValues {
341437
startRangeComparisonSign = GreaterThanOrEqualsComparisonSign
342438
}
439+
440+
if uniqueKeyColumns.Len() == 2 {
441+
return buildUniqueKeyRangeEndTwoColumnViaOffset(databaseName, tableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, startRangeComparisonSign, hint)
442+
}
443+
343444
rangeStartComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeStartArgs, startRangeComparisonSign)
344445
if err != nil {
345446
return "", explodedArgs, err
@@ -393,6 +494,11 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str
393494
if includeRangeStartValues {
394495
startRangeComparisonSign = GreaterThanOrEqualsComparisonSign
395496
}
497+
498+
if uniqueKeyColumns.Len() == 2 {
499+
return buildUniqueKeyRangeEndTwoColumnViaTemptable(databaseName, tableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, startRangeComparisonSign, hint)
500+
}
501+
396502
rangeStartComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeStartArgs, startRangeComparisonSign)
397503
if err != nil {
398504
return "", explodedArgs, err
@@ -442,6 +548,206 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str
442548
return result, explodedArgs, nil
443549
}
444550

551+
type twoColumnRangeMeta struct {
552+
col1Name, col2Name string
553+
col1Val, col2Val string
554+
orderByAsc string
555+
orderByDesc string
556+
}
557+
558+
func newTwoColumnRangeMeta(uniqueKeyColumns *ColumnList) twoColumnRangeMeta {
559+
colVals := buildColumnsPreparedValues(uniqueKeyColumns)
560+
cols := uniqueKeyColumns.Columns()
561+
col1Name := EscapeName(cols[0].Name)
562+
col2Name := EscapeName(cols[1].Name)
563+
col1Asc := fmt.Sprintf("%s asc", col1Name)
564+
col2Asc := fmt.Sprintf("%s asc", col2Name)
565+
col1Desc := fmt.Sprintf("%s desc", col1Name)
566+
col2Desc := fmt.Sprintf("%s desc", col2Name)
567+
if cols[0].Type == EnumColumnType {
568+
col1Asc = fmt.Sprintf("concat(%s) asc", col1Name)
569+
col1Desc = fmt.Sprintf("concat(%s) desc", col1Name)
570+
}
571+
if cols[1].Type == EnumColumnType {
572+
col2Asc = fmt.Sprintf("concat(%s) asc", col2Name)
573+
col2Desc = fmt.Sprintf("concat(%s) desc", col2Name)
574+
}
575+
return twoColumnRangeMeta{
576+
col1Name: col1Name,
577+
col2Name: col2Name,
578+
col1Val: colVals[0],
579+
col2Val: colVals[1],
580+
orderByAsc: col1Asc + ", " + col2Asc,
581+
orderByDesc: col1Desc + ", " + col2Desc,
582+
}
583+
}
584+
585+
func buildTwoColumnUnionParts(
586+
selectClause, fromClause string,
587+
col1Name, col2Name string,
588+
col1StartVal, col2StartVal, col1EndVal, col2EndVal string,
589+
col2StartOp, partSuffix string,
590+
rangeStartArgs, rangeEndArgs []interface{},
591+
) (part1, part2, part3 string, explodedArgs []interface{}) {
592+
part1 = fmt.Sprintf(
593+
`(select %s from %s where %s = %s and %s %s %s %s)`,
594+
selectClause, fromClause,
595+
col1Name, col1StartVal, col2Name, col2StartOp, col2StartVal,
596+
partSuffix,
597+
)
598+
explodedArgs = append(explodedArgs, rangeStartArgs[0], rangeStartArgs[1])
599+
600+
part2 = fmt.Sprintf(
601+
`(select %s from %s where %s > %s and %s < %s %s)`,
602+
selectClause, fromClause,
603+
col1Name, col1StartVal, col1Name, col1EndVal,
604+
partSuffix,
605+
)
606+
explodedArgs = append(explodedArgs, rangeStartArgs[0], rangeEndArgs[0])
607+
608+
part3 = fmt.Sprintf(
609+
`(select %s from %s where %s = %s and %s <= %s %s)`,
610+
selectClause, fromClause,
611+
col1Name, col1EndVal, col2Name, col2EndVal,
612+
partSuffix,
613+
)
614+
explodedArgs = append(explodedArgs, rangeEndArgs[0], rangeEndArgs[1])
615+
return
616+
}
617+
618+
func buildUniqueKeyRangeEndTwoColumnViaOffset(
619+
databaseName, tableName string,
620+
uniqueKeyColumns *ColumnList,
621+
rangeStartArgs, rangeEndArgs []interface{},
622+
chunkSize int64,
623+
startRangeComparisonSign ValueComparisonSign,
624+
hint string,
625+
) (result string, explodedArgs []interface{}, err error) {
626+
m := newTwoColumnRangeMeta(uniqueKeyColumns)
627+
if len(rangeStartArgs) != 2 || len(rangeEndArgs) != 2 {
628+
return "", nil, fmt.Errorf("expected 2 range args in buildUniqueKeyRangeEndTwoColumnViaOffset, got %d start and %d end", len(rangeStartArgs), len(rangeEndArgs))
629+
}
630+
col2StartOp := string(startRangeComparisonSign)
631+
selectClause := m.col1Name + ", " + m.col2Name
632+
fromClause := databaseName + "." + tableName
633+
partSuffix := fmt.Sprintf("order by %s limit %d", m.orderByAsc, chunkSize)
634+
635+
if sameFirstColumnValue(rangeStartArgs, rangeEndArgs) {
636+
result = fmt.Sprintf(`
637+
select /* gh-ost %s.%s %s */
638+
%s, %s
639+
from
640+
%s.%s
641+
where
642+
(%s = %s and %s %s %s and %s <= %s)
643+
order by
644+
%s
645+
limit 1
646+
offset %d`,
647+
databaseName, tableName, hint,
648+
m.col1Name, m.col2Name,
649+
databaseName, tableName,
650+
m.col1Name, m.col1Val, m.col2Name, col2StartOp, m.col2Val, m.col2Name, m.col2Val,
651+
m.orderByAsc,
652+
chunkSize-1,
653+
)
654+
explodedArgs = append(explodedArgs, rangeStartArgs[0], rangeStartArgs[1], rangeEndArgs[1])
655+
return result, explodedArgs, nil
656+
}
657+
658+
part1, part2, part3, explodedArgs := buildTwoColumnUnionParts(
659+
selectClause, fromClause,
660+
m.col1Name, m.col2Name,
661+
m.col1Val, m.col2Val, m.col1Val, m.col2Val,
662+
col2StartOp, partSuffix,
663+
rangeStartArgs, rangeEndArgs,
664+
)
665+
666+
result = fmt.Sprintf(`
667+
select /* gh-ost %s.%s %s */
668+
%s, %s
669+
from
670+
(%s union all %s union all %s) t
671+
order by
672+
%s
673+
limit 1
674+
offset %d`,
675+
databaseName, tableName, hint,
676+
m.col1Name, m.col2Name,
677+
part1, part2, part3,
678+
m.orderByAsc,
679+
chunkSize-1,
680+
)
681+
return result, explodedArgs, nil
682+
}
683+
684+
func buildUniqueKeyRangeEndTwoColumnViaTemptable(
685+
databaseName, tableName string,
686+
uniqueKeyColumns *ColumnList,
687+
rangeStartArgs, rangeEndArgs []interface{},
688+
chunkSize int64,
689+
startRangeComparisonSign ValueComparisonSign,
690+
hint string,
691+
) (result string, explodedArgs []interface{}, err error) {
692+
m := newTwoColumnRangeMeta(uniqueKeyColumns)
693+
if len(rangeStartArgs) != 2 || len(rangeEndArgs) != 2 {
694+
return "", nil, fmt.Errorf("expected 2 range args in buildUniqueKeyRangeEndTwoColumnViaTemptable, got %d start and %d end", len(rangeStartArgs), len(rangeEndArgs))
695+
}
696+
col2StartOp := string(startRangeComparisonSign)
697+
selectClause := m.col1Name + ", " + m.col2Name
698+
fromClause := databaseName + "." + tableName
699+
partSuffix := fmt.Sprintf("order by %s limit %d", m.orderByAsc, chunkSize)
700+
701+
if sameFirstColumnValue(rangeStartArgs, rangeEndArgs) {
702+
result = fmt.Sprintf(`
703+
select /* gh-ost %s.%s %s */ %s, %s
704+
from (
705+
select %s, %s
706+
from %s.%s
707+
where (%s = %s and %s %s %s and %s <= %s)
708+
order by %s
709+
limit %d
710+
) select_osc_chunk
711+
order by %s
712+
limit 1`,
713+
databaseName, tableName, hint, m.col1Name, m.col2Name,
714+
m.col1Name, m.col2Name,
715+
databaseName, tableName,
716+
m.col1Name, m.col1Val, m.col2Name, col2StartOp, m.col2Val, m.col2Name, m.col2Val,
717+
m.orderByAsc, chunkSize,
718+
m.orderByDesc,
719+
)
720+
explodedArgs = append(explodedArgs, rangeStartArgs[0], rangeStartArgs[1], rangeEndArgs[1])
721+
return result, explodedArgs, nil
722+
}
723+
724+
part1, part2, part3, explodedArgs := buildTwoColumnUnionParts(
725+
selectClause, fromClause,
726+
m.col1Name, m.col2Name,
727+
m.col1Val, m.col2Val, m.col1Val, m.col2Val,
728+
col2StartOp, partSuffix,
729+
rangeStartArgs, rangeEndArgs,
730+
)
731+
732+
result = fmt.Sprintf(`
733+
select /* gh-ost %s.%s %s */ %s, %s
734+
from (
735+
select %s, %s
736+
from (%s union all %s union all %s) t
737+
order by %s
738+
limit %d
739+
) select_osc_chunk
740+
order by %s
741+
limit 1`,
742+
databaseName, tableName, hint, m.col1Name, m.col2Name,
743+
m.col1Name, m.col2Name,
744+
part1, part2, part3,
745+
m.orderByAsc, chunkSize,
746+
m.orderByDesc,
747+
)
748+
return result, explodedArgs, nil
749+
}
750+
445751
func BuildUniqueKeyMinValuesPreparedQuery(databaseName, tableName string, uniqueKey *UniqueKey) (string, error) {
446752
return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, uniqueKey, "asc")
447753
}

0 commit comments

Comments
 (0)