From 0bfbf6a256348b1543e638c7d7b2f3004b289fdb Mon Sep 17 00:00:00 2001 From: Ralf Baechle Date: Thu, 21 Mar 2013 11:28:10 +0100 Subject: MIPS: Make declarations and definitions of tlbmiss_handler_setup_pgd match. tlbmiss_handler_setup_pgd is run-time generated code and it was convenient to pretend the symbol was an array in the generator but a function for the users. LTO gcc won't tolerate this kind of lie anymore so solve the problem through a cast and function pointer instead. Signed-off-by: Ralf Baechle --- arch/mips/mm/tlbex.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'arch/mips/mm') diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 820e6612d744..b71affde3e7b 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -1458,17 +1458,17 @@ u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; #ifdef CONFIG_MIPS_PGD_C0_CONTEXT -u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned; +u32 tlbmiss_handler_setup_pgd_array[16] __cacheline_aligned; static void __cpuinit build_r4000_setup_pgd(void) { const int a0 = 4; const int a1 = 5; - u32 *p = tlbmiss_handler_setup_pgd; + u32 *p = tlbmiss_handler_setup_pgd_array; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; - memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd)); + memset(tlbmiss_handler_setup_pgd_array, 0, sizeof(tlbmiss_handler_setup_pgd_array)); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); @@ -1496,15 +1496,15 @@ static void __cpuinit build_r4000_setup_pgd(void) uasm_i_jr(&p, 31); UASM_i_MTC0(&p, a0, 31, pgd_reg); } - if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd)) - panic("tlbmiss_handler_setup_pgd space exceeded"); + if (p - tlbmiss_handler_setup_pgd_array > ARRAY_SIZE(tlbmiss_handler_setup_pgd_array)) + panic("tlbmiss_handler_setup_pgd_array space exceeded"); uasm_resolve_relocs(relocs, labels); - pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", - (unsigned int)(p - tlbmiss_handler_setup_pgd)); + pr_debug("Wrote tlbmiss_handler_setup_pgd_array (%u instructions).\n", + (unsigned int)(p - tlbmiss_handler_setup_pgd_array)); dump_handler("tlbmiss_handler", - tlbmiss_handler_setup_pgd, - ARRAY_SIZE(tlbmiss_handler_setup_pgd)); + tlbmiss_handler_setup_pgd_array, + ARRAY_SIZE(tlbmiss_handler_setup_pgd_array)); } #endif @@ -2207,7 +2207,7 @@ void __cpuinit flush_tlb_handlers(void) local_flush_icache_range((unsigned long)handle_tlbm, (unsigned long)handle_tlbm + sizeof(handle_tlbm)); #ifdef CONFIG_MIPS_PGD_C0_CONTEXT - local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, - (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm)); + local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd_array, + (unsigned long)tlbmiss_handler_setup_pgd_array + sizeof(handle_tlbm)); #endif } -- cgit v1.2.3 From abc597fe623cfd7d3b18d5235c54f3d567d2c3d3 Mon Sep 17 00:00:00 2001 From: "Steven J. Hill" Date: Tue, 5 Feb 2013 16:52:01 -0600 Subject: MIPS: microMIPS: uasm: Split 'uasm.c' into two files. Split 'uasm.c' into two files. The new file 'uasm-mips.c' has the functions specific to the classic MIPS ISA. The 'uasm.c' file contains common code that can be used by classic or other ISAs that could be supported by the kernel. Signed-off-by: Steven J. Hill Cc: linux-mips@linux-mips.org Cc: cernekee@gmail.com Cc: kevink@paralogos.com Cc: ddaney.cavm@gmail.com Patchwork: https://patchwork.linux-mips.org/patch/4922/ Signed-off-by: Ralf Baechle (cherry picked from commit 0961103562ab958fa74f35043bf4f72e51ed6155) --- arch/mips/include/asm/uasm.h | 62 ++++---- arch/mips/mm/Makefile | 2 +- arch/mips/mm/uasm-mips.c | 196 ++++++++++++++++++++++++++ arch/mips/mm/uasm.c | 326 +++++++++++-------------------------------- 4 files changed, 313 insertions(+), 273 deletions(-) create mode 100644 arch/mips/mm/uasm-mips.c (limited to 'arch/mips/mm') diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h index 058e941626a6..f7d8f1509c4d 100644 --- a/arch/mips/include/asm/uasm.h +++ b/arch/mips/include/asm/uasm.h @@ -6,7 +6,7 @@ * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer * Copyright (C) 2005 Maciej W. Rozycki * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) - * Copyright (C) 2012 MIPS Technologies, Inc. + * Copyright (C) 2012, 2013 MIPS Technologies, Inc. All rights reserved. */ #include @@ -22,44 +22,57 @@ #define UASM_EXPORT_SYMBOL(sym) #endif +#define _UASM_ISA_CLASSIC 0 + +#ifndef UASM_ISA +#define UASM_ISA _UASM_ISA_CLASSIC +#endif + +#if (UASM_ISA == _UASM_ISA_CLASSIC) +#define ISAOPC(op) uasm_i##op +#define ISAFUNC(x) x +#else +#error Unsupported micro-assembler ISA!!! +#endif + #define Ip_u1u2u3(op) \ void __uasminit \ -uasm_i##op(u32 **buf, unsigned int a, unsigned int b, unsigned int c) +ISAOPC(op)(u32 **buf, unsigned int a, unsigned int b, unsigned int c) #define Ip_u2u1u3(op) \ void __uasminit \ -uasm_i##op(u32 **buf, unsigned int a, unsigned int b, unsigned int c) +ISAOPC(op)(u32 **buf, unsigned int a, unsigned int b, unsigned int c) #define Ip_u3u1u2(op) \ void __uasminit \ -uasm_i##op(u32 **buf, unsigned int a, unsigned int b, unsigned int c) +ISAOPC(op)(u32 **buf, unsigned int a, unsigned int b, unsigned int c) #define Ip_u1u2s3(op) \ void __uasminit \ -uasm_i##op(u32 **buf, unsigned int a, unsigned int b, signed int c) +ISAOPC(op)(u32 **buf, unsigned int a, unsigned int b, signed int c) #define Ip_u2s3u1(op) \ void __uasminit \ -uasm_i##op(u32 **buf, unsigned int a, signed int b, unsigned int c) +ISAOPC(op)(u32 **buf, unsigned int a, signed int b, unsigned int c) #define Ip_u2u1s3(op) \ void __uasminit \ -uasm_i##op(u32 **buf, unsigned int a, unsigned int b, signed int c) +ISAOPC(op)(u32 **buf, unsigned int a, unsigned int b, signed int c) #define Ip_u2u1msbu3(op) \ void __uasminit \ -uasm_i##op(u32 **buf, unsigned int a, unsigned int b, unsigned int c, \ +ISAOPC(op)(u32 **buf, unsigned int a, unsigned int b, unsigned int c, \ unsigned int d) #define Ip_u1u2(op) \ -void __uasminit uasm_i##op(u32 **buf, unsigned int a, unsigned int b) +void __uasminit ISAOPC(op)(u32 **buf, unsigned int a, unsigned int b) #define Ip_u1s2(op) \ -void __uasminit uasm_i##op(u32 **buf, unsigned int a, signed int b) +void __uasminit ISAOPC(op)(u32 **buf, unsigned int a, signed int b) -#define Ip_u1(op) void __uasminit uasm_i##op(u32 **buf, unsigned int a) +#define Ip_u1(op) void __uasminit ISAOPC(op)(u32 **buf, unsigned int a) -#define Ip_0(op) void __uasminit uasm_i##op(u32 **buf) +#define Ip_0(op) void __uasminit ISAOPC(op)(u32 **buf) Ip_u2u1s3(_addiu); Ip_u3u1u2(_addu); @@ -132,14 +145,15 @@ struct uasm_label { int lab; }; -void __uasminit uasm_build_label(struct uasm_label **lab, u32 *addr, int lid); +void __uasminit ISAFUNC(uasm_build_label)(struct uasm_label **lab, u32 *addr, + int lid); #ifdef CONFIG_64BIT -int uasm_in_compat_space_p(long addr); +int ISAFUNC(uasm_in_compat_space_p)(long addr); #endif -int uasm_rel_hi(long val); -int uasm_rel_lo(long val); -void UASM_i_LA_mostly(u32 **buf, unsigned int rs, long addr); -void UASM_i_LA(u32 **buf, unsigned int rs, long addr); +int ISAFUNC(uasm_rel_hi)(long val); +int ISAFUNC(uasm_rel_lo)(long val); +void ISAFUNC(UASM_i_LA_mostly)(u32 **buf, unsigned int rs, long addr); +void ISAFUNC(UASM_i_LA)(u32 **buf, unsigned int rs, long addr); #define UASM_L_LA(lb) \ static inline void __uasminit uasm_l##lb(struct uasm_label **lab, u32 *addr) \ @@ -196,27 +210,27 @@ static inline void uasm_i_drotr_safe(u32 **p, unsigned int a1, unsigned int a2, unsigned int a3) { if (a3 < 32) - uasm_i_drotr(p, a1, a2, a3); + ISAOPC(_drotr)(p, a1, a2, a3); else - uasm_i_drotr32(p, a1, a2, a3 - 32); + ISAOPC(_drotr32)(p, a1, a2, a3 - 32); } static inline void uasm_i_dsll_safe(u32 **p, unsigned int a1, unsigned int a2, unsigned int a3) { if (a3 < 32) - uasm_i_dsll(p, a1, a2, a3); + ISAOPC(_dsll)(p, a1, a2, a3); else - uasm_i_dsll32(p, a1, a2, a3 - 32); + ISAOPC(_dsll32)(p, a1, a2, a3 - 32); } static inline void uasm_i_dsrl_safe(u32 **p, unsigned int a1, unsigned int a2, unsigned int a3) { if (a3 < 32) - uasm_i_dsrl(p, a1, a2, a3); + ISAOPC(_dsrl)(p, a1, a2, a3); else - uasm_i_dsrl32(p, a1, a2, a3 - 32); + ISAOPC(_dsrl32)(p, a1, a2, a3 - 32); } /* Handle relocations. */ diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile index 1dcec30ad1c4..9e90c2182aa5 100644 --- a/arch/mips/mm/Makefile +++ b/arch/mips/mm/Makefile @@ -4,7 +4,7 @@ obj-y += cache.o dma-default.o extable.o fault.o \ gup.o init.o mmap.o page.o page-funcs.o \ - tlbex.o tlbex-fault.o uasm.o + tlbex.o tlbex-fault.o uasm-mips.o obj-$(CONFIG_32BIT) += ioremap.o pgtable-32.o obj-$(CONFIG_64BIT) += pgtable-64.o diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c new file mode 100644 index 000000000000..e78e74dc3aea --- /dev/null +++ b/arch/mips/mm/uasm-mips.c @@ -0,0 +1,196 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * A small micro-assembler. It is intentionally kept simple, does only + * support a subset of instructions, and does not try to hide pipeline + * effects like branch delay slots. + * + * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer + * Copyright (C) 2005, 2007 Maciej W. Rozycki + * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) + * Copyright (C) 2012, 2013 MIPS Technologies, Inc. All rights reserved. + */ + +#include +#include +#include + +#include +#include +#include +#include + +#define RS_MASK 0x1f +#define RS_SH 21 +#define RT_MASK 0x1f +#define RT_SH 16 +#define SCIMM_MASK 0xfffff +#define SCIMM_SH 6 + +/* This macro sets the non-variable bits of an instruction. */ +#define M(a, b, c, d, e, f) \ + ((a) << OP_SH \ + | (b) << RS_SH \ + | (c) << RT_SH \ + | (d) << RD_SH \ + | (e) << RE_SH \ + | (f) << FUNC_SH) + +#include "uasm.c" + +static struct insn insn_table[] __uasminitdata = { + { insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD }, + { insn_andi, M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM }, + { insn_and, M(spec_op, 0, 0, 0, 0, and_op), RS | RT | RD }, + { insn_bbit0, M(lwc2_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, + { insn_bbit1, M(swc2_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, + { insn_beql, M(beql_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, + { insn_beq, M(beq_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, + { insn_bgezl, M(bcond_op, 0, bgezl_op, 0, 0, 0), RS | BIMM }, + { insn_bgez, M(bcond_op, 0, bgez_op, 0, 0, 0), RS | BIMM }, + { insn_bltzl, M(bcond_op, 0, bltzl_op, 0, 0, 0), RS | BIMM }, + { insn_bltz, M(bcond_op, 0, bltz_op, 0, 0, 0), RS | BIMM }, + { insn_bne, M(bne_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, + { insn_cache, M(cache_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_daddiu, M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_daddu, M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD }, + { insn_dinsm, M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE }, + { insn_dins, M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE }, + { insn_dmfc0, M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET}, + { insn_dmtc0, M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET}, + { insn_drotr32, M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE }, + { insn_drotr, M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE }, + { insn_dsll32, M(spec_op, 0, 0, 0, 0, dsll32_op), RT | RD | RE }, + { insn_dsll, M(spec_op, 0, 0, 0, 0, dsll_op), RT | RD | RE }, + { insn_dsra, M(spec_op, 0, 0, 0, 0, dsra_op), RT | RD | RE }, + { insn_dsrl32, M(spec_op, 0, 0, 0, 0, dsrl32_op), RT | RD | RE }, + { insn_dsrl, M(spec_op, 0, 0, 0, 0, dsrl_op), RT | RD | RE }, + { insn_dsubu, M(spec_op, 0, 0, 0, 0, dsubu_op), RS | RT | RD }, + { insn_eret, M(cop0_op, cop_op, 0, 0, 0, eret_op), 0 }, + { insn_ext, M(spec3_op, 0, 0, 0, 0, ext_op), RS | RT | RD | RE }, + { insn_ins, M(spec3_op, 0, 0, 0, 0, ins_op), RS | RT | RD | RE }, + { insn_j, M(j_op, 0, 0, 0, 0, 0), JIMM }, + { insn_jal, M(jal_op, 0, 0, 0, 0, 0), JIMM }, + { insn_j, M(j_op, 0, 0, 0, 0, 0), JIMM }, + { insn_jr, M(spec_op, 0, 0, 0, 0, jr_op), RS }, + { insn_ld, M(ld_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_ldx, M(spec3_op, 0, 0, 0, ldx_op, lx_op), RS | RT | RD }, + { insn_lld, M(lld_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_ll, M(ll_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_lui, M(lui_op, 0, 0, 0, 0, 0), RT | SIMM }, + { insn_lw, M(lw_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_lwx, M(spec3_op, 0, 0, 0, lwx_op, lx_op), RS | RT | RD }, + { insn_mfc0, M(cop0_op, mfc_op, 0, 0, 0, 0), RT | RD | SET}, + { insn_mtc0, M(cop0_op, mtc_op, 0, 0, 0, 0), RT | RD | SET}, + { insn_ori, M(ori_op, 0, 0, 0, 0, 0), RS | RT | UIMM }, + { insn_or, M(spec_op, 0, 0, 0, 0, or_op), RS | RT | RD }, + { insn_pref, M(pref_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_rfe, M(cop0_op, cop_op, 0, 0, 0, rfe_op), 0 }, + { insn_rotr, M(spec_op, 1, 0, 0, 0, srl_op), RT | RD | RE }, + { insn_scd, M(scd_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_sc, M(sc_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_sd, M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_sll, M(spec_op, 0, 0, 0, 0, sll_op), RT | RD | RE }, + { insn_sra, M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE }, + { insn_srl, M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE }, + { insn_subu, M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD }, + { insn_sw, M(sw_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, + { insn_syscall, M(spec_op, 0, 0, 0, 0, syscall_op), SCIMM}, + { insn_tlbp, M(cop0_op, cop_op, 0, 0, 0, tlbp_op), 0 }, + { insn_tlbr, M(cop0_op, cop_op, 0, 0, 0, tlbr_op), 0 }, + { insn_tlbwi, M(cop0_op, cop_op, 0, 0, 0, tlbwi_op), 0 }, + { insn_tlbwr, M(cop0_op, cop_op, 0, 0, 0, tlbwr_op), 0 }, + { insn_xori, M(xori_op, 0, 0, 0, 0, 0), RS | RT | UIMM }, + { insn_xor, M(spec_op, 0, 0, 0, 0, xor_op), RS | RT | RD }, + { insn_invalid, 0, 0 } +}; + +#undef M + +static inline __uasminit u32 build_bimm(s32 arg) +{ + WARN(arg > 0x1ffff || arg < -0x20000, + KERN_WARNING "Micro-assembler field overflow\n"); + + WARN(arg & 0x3, KERN_WARNING "Invalid micro-assembler branch target\n"); + + return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff); +} + +static inline __uasminit u32 build_jimm(u32 arg) +{ + WARN(arg & ~(JIMM_MASK << 2), + KERN_WARNING "Micro-assembler field overflow\n"); + + return (arg >> 2) & JIMM_MASK; +} + +/* + * The order of opcode arguments is implicitly left to right, + * starting with RS and ending with FUNC or IMM. + */ +static void __uasminit build_insn(u32 **buf, enum opcode opc, ...) +{ + struct insn *ip = NULL; + unsigned int i; + va_list ap; + u32 op; + + for (i = 0; insn_table[i].opcode != insn_invalid; i++) + if (insn_table[i].opcode == opc) { + ip = &insn_table[i]; + break; + } + + if (!ip || (opc == insn_daddiu && r4k_daddiu_bug())) + panic("Unsupported Micro-assembler instruction %d", opc); + + op = ip->match; + va_start(ap, opc); + if (ip->fields & RS) + op |= build_rs(va_arg(ap, u32)); + if (ip->fields & RT) + op |= build_rt(va_arg(ap, u32)); + if (ip->fields & RD) + op |= build_rd(va_arg(ap, u32)); + if (ip->fields & RE) + op |= build_re(va_arg(ap, u32)); + if (ip->fields & SIMM) + op |= build_simm(va_arg(ap, s32)); + if (ip->fields & UIMM) + op |= build_uimm(va_arg(ap, u32)); + if (ip->fields & BIMM) + op |= build_bimm(va_arg(ap, s32)); + if (ip->fields & JIMM) + op |= build_jimm(va_arg(ap, u32)); + if (ip->fields & FUNC) + op |= build_func(va_arg(ap, u32)); + if (ip->fields & SET) + op |= build_set(va_arg(ap, u32)); + if (ip->fields & SCIMM) + op |= build_scimm(va_arg(ap, u32)); + va_end(ap); + + **buf = op; + (*buf)++; +} + +static inline void __uasminit +__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab) +{ + long laddr = (long)lab->addr; + long raddr = (long)rel->addr; + + switch (rel->type) { + case R_MIPS_PC16: + *rel->addr |= build_bimm(laddr - (raddr + 4)); + break; + + default: + panic("Unsupported Micro-assembler relocation %d", + rel->type); + } +} diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c index 942ff6c2eba2..7eb5e4355d25 100644 --- a/arch/mips/mm/uasm.c +++ b/arch/mips/mm/uasm.c @@ -10,17 +10,9 @@ * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer * Copyright (C) 2005, 2007 Maciej W. Rozycki * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) + * Copyright (C) 2012, 2013 MIPS Technologies, Inc. All rights reserved. */ -#include -#include -#include - -#include -#include -#include -#include - enum fields { RS = 0x001, RT = 0x002, @@ -37,10 +29,6 @@ enum fields { #define OP_MASK 0x3f #define OP_SH 26 -#define RS_MASK 0x1f -#define RS_SH 21 -#define RT_MASK 0x1f -#define RT_SH 16 #define RD_MASK 0x1f #define RD_SH 11 #define RE_MASK 0x1f @@ -53,8 +41,6 @@ enum fields { #define FUNC_SH 0 #define SET_MASK 0x7 #define SET_SH 0 -#define SCIMM_MASK 0xfffff -#define SCIMM_SH 6 enum opcode { insn_invalid, @@ -77,85 +63,6 @@ struct insn { enum fields fields; }; -/* This macro sets the non-variable bits of an instruction. */ -#define M(a, b, c, d, e, f) \ - ((a) << OP_SH \ - | (b) << RS_SH \ - | (c) << RT_SH \ - | (d) << RD_SH \ - | (e) << RE_SH \ - | (f) << FUNC_SH) - -static struct insn insn_table[] __uasminitdata = { - { insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD }, - { insn_andi, M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM }, - { insn_and, M(spec_op, 0, 0, 0, 0, and_op), RS | RT | RD }, - { insn_bbit0, M(lwc2_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, - { insn_bbit1, M(swc2_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, - { insn_beql, M(beql_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, - { insn_beq, M(beq_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, - { insn_bgezl, M(bcond_op, 0, bgezl_op, 0, 0, 0), RS | BIMM }, - { insn_bgez, M(bcond_op, 0, bgez_op, 0, 0, 0), RS | BIMM }, - { insn_bltzl, M(bcond_op, 0, bltzl_op, 0, 0, 0), RS | BIMM }, - { insn_bltz, M(bcond_op, 0, bltz_op, 0, 0, 0), RS | BIMM }, - { insn_bne, M(bne_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, - { insn_cache, M(cache_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_daddiu, M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_daddu, M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD }, - { insn_dinsm, M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE }, - { insn_dins, M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE }, - { insn_dmfc0, M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET}, - { insn_dmtc0, M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET}, - { insn_drotr32, M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE }, - { insn_drotr, M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE }, - { insn_dsll32, M(spec_op, 0, 0, 0, 0, dsll32_op), RT | RD | RE }, - { insn_dsll, M(spec_op, 0, 0, 0, 0, dsll_op), RT | RD | RE }, - { insn_dsra, M(spec_op, 0, 0, 0, 0, dsra_op), RT | RD | RE }, - { insn_dsrl32, M(spec_op, 0, 0, 0, 0, dsrl32_op), RT | RD | RE }, - { insn_dsrl, M(spec_op, 0, 0, 0, 0, dsrl_op), RT | RD | RE }, - { insn_dsubu, M(spec_op, 0, 0, 0, 0, dsubu_op), RS | RT | RD }, - { insn_eret, M(cop0_op, cop_op, 0, 0, 0, eret_op), 0 }, - { insn_ext, M(spec3_op, 0, 0, 0, 0, ext_op), RS | RT | RD | RE }, - { insn_ins, M(spec3_op, 0, 0, 0, 0, ins_op), RS | RT | RD | RE }, - { insn_j, M(j_op, 0, 0, 0, 0, 0), JIMM }, - { insn_jal, M(jal_op, 0, 0, 0, 0, 0), JIMM }, - { insn_j, M(j_op, 0, 0, 0, 0, 0), JIMM }, - { insn_jr, M(spec_op, 0, 0, 0, 0, jr_op), RS }, - { insn_ld, M(ld_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_ldx, M(spec3_op, 0, 0, 0, ldx_op, lx_op), RS | RT | RD }, - { insn_lld, M(lld_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_ll, M(ll_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_lui, M(lui_op, 0, 0, 0, 0, 0), RT | SIMM }, - { insn_lw, M(lw_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_lwx, M(spec3_op, 0, 0, 0, lwx_op, lx_op), RS | RT | RD }, - { insn_mfc0, M(cop0_op, mfc_op, 0, 0, 0, 0), RT | RD | SET}, - { insn_mtc0, M(cop0_op, mtc_op, 0, 0, 0, 0), RT | RD | SET}, - { insn_ori, M(ori_op, 0, 0, 0, 0, 0), RS | RT | UIMM }, - { insn_or, M(spec_op, 0, 0, 0, 0, or_op), RS | RT | RD }, - { insn_pref, M(pref_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_rfe, M(cop0_op, cop_op, 0, 0, 0, rfe_op), 0 }, - { insn_rotr, M(spec_op, 1, 0, 0, 0, srl_op), RT | RD | RE }, - { insn_scd, M(scd_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_sc, M(sc_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_sd, M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_sll, M(spec_op, 0, 0, 0, 0, sll_op), RT | RD | RE }, - { insn_sra, M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE }, - { insn_srl, M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE }, - { insn_subu, M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD }, - { insn_sw, M(sw_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, - { insn_syscall, M(spec_op, 0, 0, 0, 0, syscall_op), SCIMM}, - { insn_tlbp, M(cop0_op, cop_op, 0, 0, 0, tlbp_op), 0 }, - { insn_tlbr, M(cop0_op, cop_op, 0, 0, 0, tlbr_op), 0 }, - { insn_tlbwi, M(cop0_op, cop_op, 0, 0, 0, tlbwi_op), 0 }, - { insn_tlbwr, M(cop0_op, cop_op, 0, 0, 0, tlbwr_op), 0 }, - { insn_xori, M(xori_op, 0, 0, 0, 0, 0), RS | RT | UIMM }, - { insn_xor, M(spec_op, 0, 0, 0, 0, xor_op), RS | RT | RD }, - { insn_invalid, 0, 0 } -}; - -#undef M - static inline __uasminit u32 build_rs(u32 arg) { WARN(arg & ~RS_MASK, KERN_WARNING "Micro-assembler field overflow\n"); @@ -199,24 +106,6 @@ static inline __uasminit u32 build_uimm(u32 arg) return arg & IMM_MASK; } -static inline __uasminit u32 build_bimm(s32 arg) -{ - WARN(arg > 0x1ffff || arg < -0x20000, - KERN_WARNING "Micro-assembler field overflow\n"); - - WARN(arg & 0x3, KERN_WARNING "Invalid micro-assembler branch target\n"); - - return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff); -} - -static inline __uasminit u32 build_jimm(u32 arg) -{ - WARN(arg & ~(JIMM_MASK << 2), - KERN_WARNING "Micro-assembler field overflow\n"); - - return (arg >> 2) & JIMM_MASK; -} - static inline __uasminit u32 build_scimm(u32 arg) { WARN(arg & ~SCIMM_MASK, @@ -239,55 +128,7 @@ static inline __uasminit u32 build_set(u32 arg) return arg & SET_MASK; } -/* - * The order of opcode arguments is implicitly left to right, - * starting with RS and ending with FUNC or IMM. - */ -static void __uasminit build_insn(u32 **buf, enum opcode opc, ...) -{ - struct insn *ip = NULL; - unsigned int i; - va_list ap; - u32 op; - - for (i = 0; insn_table[i].opcode != insn_invalid; i++) - if (insn_table[i].opcode == opc) { - ip = &insn_table[i]; - break; - } - - if (!ip || (opc == insn_daddiu && r4k_daddiu_bug())) - panic("Unsupported Micro-assembler instruction %d", opc); - - op = ip->match; - va_start(ap, opc); - if (ip->fields & RS) - op |= build_rs(va_arg(ap, u32)); - if (ip->fields & RT) - op |= build_rt(va_arg(ap, u32)); - if (ip->fields & RD) - op |= build_rd(va_arg(ap, u32)); - if (ip->fields & RE) - op |= build_re(va_arg(ap, u32)); - if (ip->fields & SIMM) - op |= build_simm(va_arg(ap, s32)); - if (ip->fields & UIMM) - op |= build_uimm(va_arg(ap, u32)); - if (ip->fields & BIMM) - op |= build_bimm(va_arg(ap, s32)); - if (ip->fields & JIMM) - op |= build_jimm(va_arg(ap, u32)); - if (ip->fields & FUNC) - op |= build_func(va_arg(ap, u32)); - if (ip->fields & SET) - op |= build_set(va_arg(ap, u32)); - if (ip->fields & SCIMM) - op |= build_scimm(va_arg(ap, u32)); - va_end(ap); - - **buf = op; - (*buf)++; -} +static void __uasminit build_insn(u32 **buf, enum opcode opc, ...); #define I_u1u2u3(op) \ Ip_u1u2u3(op) \ @@ -445,7 +286,7 @@ I_u3u1u2(_ldx) #ifdef CONFIG_CPU_CAVIUM_OCTEON #include -void __uasminit uasm_i_pref(u32 **buf, unsigned int a, signed int b, +void __uasminit ISAFUNC(uasm_i_pref)(u32 **buf, unsigned int a, signed int b, unsigned int c) { if (OCTEON_IS_MODEL(OCTEON_CN63XX_PASS1_X) && a <= 24 && a != 5) @@ -457,21 +298,21 @@ void __uasminit uasm_i_pref(u32 **buf, unsigned int a, signed int b, else build_insn(buf, insn_pref, c, a, b); } -UASM_EXPORT_SYMBOL(uasm_i_pref); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_i_pref)); #else I_u2s3u1(_pref) #endif /* Handle labels. */ -void __uasminit uasm_build_label(struct uasm_label **lab, u32 *addr, int lid) +void __uasminit ISAFUNC(uasm_build_label)(struct uasm_label **lab, u32 *addr, int lid) { (*lab)->addr = addr; (*lab)->lab = lid; (*lab)++; } -UASM_EXPORT_SYMBOL(uasm_build_label); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_build_label)); -int __uasminit uasm_in_compat_space_p(long addr) +int __uasminit ISAFUNC(uasm_in_compat_space_p)(long addr) { /* Is this address in 32bit compat space? */ #ifdef CONFIG_64BIT @@ -480,7 +321,7 @@ int __uasminit uasm_in_compat_space_p(long addr) return 1; #endif } -UASM_EXPORT_SYMBOL(uasm_in_compat_space_p); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_in_compat_space_p)); static int __uasminit uasm_rel_highest(long val) { @@ -500,77 +341,66 @@ static int __uasminit uasm_rel_higher(long val) #endif } -int __uasminit uasm_rel_hi(long val) +int __uasminit ISAFUNC(uasm_rel_hi)(long val) { return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000; } -UASM_EXPORT_SYMBOL(uasm_rel_hi); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_rel_hi)); -int __uasminit uasm_rel_lo(long val) +int __uasminit ISAFUNC(uasm_rel_lo)(long val) { return ((val & 0xffff) ^ 0x8000) - 0x8000; } -UASM_EXPORT_SYMBOL(uasm_rel_lo); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_rel_lo)); -void __uasminit UASM_i_LA_mostly(u32 **buf, unsigned int rs, long addr) +void __uasminit ISAFUNC(UASM_i_LA_mostly)(u32 **buf, unsigned int rs, long addr) { - if (!uasm_in_compat_space_p(addr)) { - uasm_i_lui(buf, rs, uasm_rel_highest(addr)); + if (!ISAFUNC(uasm_in_compat_space_p)(addr)) { + ISAFUNC(uasm_i_lui)(buf, rs, uasm_rel_highest(addr)); if (uasm_rel_higher(addr)) - uasm_i_daddiu(buf, rs, rs, uasm_rel_higher(addr)); - if (uasm_rel_hi(addr)) { - uasm_i_dsll(buf, rs, rs, 16); - uasm_i_daddiu(buf, rs, rs, uasm_rel_hi(addr)); - uasm_i_dsll(buf, rs, rs, 16); + ISAFUNC(uasm_i_daddiu)(buf, rs, rs, uasm_rel_higher(addr)); + if (ISAFUNC(uasm_rel_hi(addr))) { + ISAFUNC(uasm_i_dsll)(buf, rs, rs, 16); + ISAFUNC(uasm_i_daddiu)(buf, rs, rs, + ISAFUNC(uasm_rel_hi)(addr)); + ISAFUNC(uasm_i_dsll)(buf, rs, rs, 16); } else - uasm_i_dsll32(buf, rs, rs, 0); + ISAFUNC(uasm_i_dsll32)(buf, rs, rs, 0); } else - uasm_i_lui(buf, rs, uasm_rel_hi(addr)); + ISAFUNC(uasm_i_lui)(buf, rs, ISAFUNC(uasm_rel_hi(addr))); } -UASM_EXPORT_SYMBOL(UASM_i_LA_mostly); +UASM_EXPORT_SYMBOL(ISAFUNC(UASM_i_LA_mostly)); -void __uasminit UASM_i_LA(u32 **buf, unsigned int rs, long addr) +void __uasminit ISAFUNC(UASM_i_LA)(u32 **buf, unsigned int rs, long addr) { - UASM_i_LA_mostly(buf, rs, addr); - if (uasm_rel_lo(addr)) { - if (!uasm_in_compat_space_p(addr)) - uasm_i_daddiu(buf, rs, rs, uasm_rel_lo(addr)); + ISAFUNC(UASM_i_LA_mostly)(buf, rs, addr); + if (ISAFUNC(uasm_rel_lo(addr))) { + if (!ISAFUNC(uasm_in_compat_space_p)(addr)) + ISAFUNC(uasm_i_daddiu)(buf, rs, rs, + ISAFUNC(uasm_rel_lo(addr))); else - uasm_i_addiu(buf, rs, rs, uasm_rel_lo(addr)); + ISAFUNC(uasm_i_addiu)(buf, rs, rs, + ISAFUNC(uasm_rel_lo(addr))); } } -UASM_EXPORT_SYMBOL(UASM_i_LA); +UASM_EXPORT_SYMBOL(ISAFUNC(UASM_i_LA)); /* Handle relocations. */ void __uasminit -uasm_r_mips_pc16(struct uasm_reloc **rel, u32 *addr, int lid) +ISAFUNC(uasm_r_mips_pc16)(struct uasm_reloc **rel, u32 *addr, int lid) { (*rel)->addr = addr; (*rel)->type = R_MIPS_PC16; (*rel)->lab = lid; (*rel)++; } -UASM_EXPORT_SYMBOL(uasm_r_mips_pc16); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_r_mips_pc16)); static inline void __uasminit -__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab) -{ - long laddr = (long)lab->addr; - long raddr = (long)rel->addr; - - switch (rel->type) { - case R_MIPS_PC16: - *rel->addr |= build_bimm(laddr - (raddr + 4)); - break; - - default: - panic("Unsupported Micro-assembler relocation %d", - rel->type); - } -} +__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab); void __uasminit -uasm_resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab) +ISAFUNC(uasm_resolve_relocs)(struct uasm_reloc *rel, struct uasm_label *lab) { struct uasm_label *l; @@ -579,40 +409,40 @@ uasm_resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab) if (rel->lab == l->lab) __resolve_relocs(rel, l); } -UASM_EXPORT_SYMBOL(uasm_resolve_relocs); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_resolve_relocs)); void __uasminit -uasm_move_relocs(struct uasm_reloc *rel, u32 *first, u32 *end, long off) +ISAFUNC(uasm_move_relocs)(struct uasm_reloc *rel, u32 *first, u32 *end, long off) { for (; rel->lab != UASM_LABEL_INVALID; rel++) if (rel->addr >= first && rel->addr < end) rel->addr += off; } -UASM_EXPORT_SYMBOL(uasm_move_relocs); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_move_relocs)); void __uasminit -uasm_move_labels(struct uasm_label *lab, u32 *first, u32 *end, long off) +ISAFUNC(uasm_move_labels)(struct uasm_label *lab, u32 *first, u32 *end, long off) { for (; lab->lab != UASM_LABEL_INVALID; lab++) if (lab->addr >= first && lab->addr < end) lab->addr += off; } -UASM_EXPORT_SYMBOL(uasm_move_labels); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_move_labels)); void __uasminit -uasm_copy_handler(struct uasm_reloc *rel, struct uasm_label *lab, u32 *first, +ISAFUNC(uasm_copy_handler)(struct uasm_reloc *rel, struct uasm_label *lab, u32 *first, u32 *end, u32 *target) { long off = (long)(target - first); memcpy(target, first, (end - first) * sizeof(u32)); - uasm_move_relocs(rel, first, end, off); - uasm_move_labels(lab, first, end, off); + ISAFUNC(uasm_move_relocs(rel, first, end, off)); + ISAFUNC(uasm_move_labels(lab, first, end, off)); } -UASM_EXPORT_SYMBOL(uasm_copy_handler); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_copy_handler)); -int __uasminit uasm_insn_has_bdelay(struct uasm_reloc *rel, u32 *addr) +int __uasminit ISAFUNC(uasm_insn_has_bdelay)(struct uasm_reloc *rel, u32 *addr) { for (; rel->lab != UASM_LABEL_INVALID; rel++) { if (rel->addr == addr @@ -623,88 +453,88 @@ int __uasminit uasm_insn_has_bdelay(struct uasm_reloc *rel, u32 *addr) return 0; } -UASM_EXPORT_SYMBOL(uasm_insn_has_bdelay); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_insn_has_bdelay)); /* Convenience functions for labeled branches. */ void __uasminit -uasm_il_bltz(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) +ISAFUNC(uasm_il_bltz)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) { uasm_r_mips_pc16(r, *p, lid); - uasm_i_bltz(p, reg, 0); + ISAFUNC(uasm_i_bltz)(p, reg, 0); } -UASM_EXPORT_SYMBOL(uasm_il_bltz); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bltz)); void __uasminit -uasm_il_b(u32 **p, struct uasm_reloc **r, int lid) +ISAFUNC(uasm_il_b)(u32 **p, struct uasm_reloc **r, int lid) { uasm_r_mips_pc16(r, *p, lid); - uasm_i_b(p, 0); + ISAFUNC(uasm_i_b)(p, 0); } -UASM_EXPORT_SYMBOL(uasm_il_b); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_b)); void __uasminit -uasm_il_beqz(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) +ISAFUNC(uasm_il_beqz)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) { uasm_r_mips_pc16(r, *p, lid); - uasm_i_beqz(p, reg, 0); + ISAFUNC(uasm_i_beqz)(p, reg, 0); } -UASM_EXPORT_SYMBOL(uasm_il_beqz); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_beqz)); void __uasminit -uasm_il_beqzl(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) +ISAFUNC(uasm_il_beqzl)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) { uasm_r_mips_pc16(r, *p, lid); - uasm_i_beqzl(p, reg, 0); + ISAFUNC(uasm_i_beqzl)(p, reg, 0); } -UASM_EXPORT_SYMBOL(uasm_il_beqzl); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_beqzl)); void __uasminit -uasm_il_bne(u32 **p, struct uasm_reloc **r, unsigned int reg1, +ISAFUNC(uasm_il_bne)(u32 **p, struct uasm_reloc **r, unsigned int reg1, unsigned int reg2, int lid) { uasm_r_mips_pc16(r, *p, lid); - uasm_i_bne(p, reg1, reg2, 0); + ISAFUNC(uasm_i_bne)(p, reg1, reg2, 0); } -UASM_EXPORT_SYMBOL(uasm_il_bne); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bne)); void __uasminit -uasm_il_bnez(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) +ISAFUNC(uasm_il_bnez)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) { uasm_r_mips_pc16(r, *p, lid); - uasm_i_bnez(p, reg, 0); + ISAFUNC(uasm_i_bnez)(p, reg, 0); } -UASM_EXPORT_SYMBOL(uasm_il_bnez); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bnez)); void __uasminit -uasm_il_bgezl(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) +ISAFUNC(uasm_il_bgezl)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) { uasm_r_mips_pc16(r, *p, lid); - uasm_i_bgezl(p, reg, 0); + ISAFUNC(uasm_i_bgezl)(p, reg, 0); } -UASM_EXPORT_SYMBOL(uasm_il_bgezl); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bgezl)); void __uasminit -uasm_il_bgez(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) +ISAFUNC(uasm_il_bgez)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid) { uasm_r_mips_pc16(r, *p, lid); - uasm_i_bgez(p, reg, 0); + ISAFUNC(uasm_i_bgez)(p, reg, 0); } -UASM_EXPORT_SYMBOL(uasm_il_bgez); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bgez)); void __uasminit -uasm_il_bbit0(u32 **p, struct uasm_reloc **r, unsigned int reg, +ISAFUNC(uasm_il_bbit0)(u32 **p, struct uasm_reloc **r, unsigned int reg, unsigned int bit, int lid) { uasm_r_mips_pc16(r, *p, lid); - uasm_i_bbit0(p, reg, bit, 0); + ISAFUNC(uasm_i_bbit0)(p, reg, bit, 0); } -UASM_EXPORT_SYMBOL(uasm_il_bbit0); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bbit0)); void __uasminit -uasm_il_bbit1(u32 **p, struct uasm_reloc **r, unsigned int reg, +ISAFUNC(uasm_il_bbit1)(u32 **p, struct uasm_reloc **r, unsigned int reg, unsigned int bit, int lid) { uasm_r_mips_pc16(r, *p, lid); - uasm_i_bbit1(p, reg, bit, 0); + ISAFUNC(uasm_i_bbit1)(p, reg, bit, 0); } -UASM_EXPORT_SYMBOL(uasm_il_bbit1); +UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bbit1)); -- cgit v1.2.3 From a6a4834cdbef23a7db16e7598c8c6e427ac82531 Mon Sep 17 00:00:00 2001 From: "Steven J. Hill" Date: Tue, 5 Feb 2013 16:52:02 -0600 Subject: MIPS: microMIPS: uasm: Add microMIPS micro assembler support. Add new file 'uasm-micromips.c' that allows the micro assembler to generate microMIPS ISA code. It can be included in the kernel alongside the classic ISA as long as the platform supports the microMIPS ISA. Signed-off-by: Steven J. Hill Cc: linux-mips@linux-mips.org Cc: cernekee@gmail.com Cc: kevink@paralogos.com Cc: ddaney.cavm@gmail.com Patchwork: https://patchwork.linux-mips.org/patch/4923/ Signed-off-by: Ralf Baechle (cherry picked from commit 5f011a866afbd03a5379f67f4e70e5efbdfc16e9) --- arch/mips/Kconfig | 4 + arch/mips/include/asm/uasm.h | 4 + arch/mips/mm/Makefile | 2 + arch/mips/mm/uasm-micromips.c | 220 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 230 insertions(+) create mode 100644 arch/mips/mm/uasm-micromips.c (limited to 'arch/mips/mm') diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 51244bf97271..03de102b7a09 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -354,6 +354,7 @@ config MIPS_SEAD3 select SYS_SUPPORTS_BIG_ENDIAN select SYS_SUPPORTS_LITTLE_ENDIAN select SYS_SUPPORTS_SMARTMIPS + select SYS_SUPPORTS_MICROMIPS select USB_ARCH_HAS_EHCI select USB_EHCI_BIG_ENDIAN_DESC select USB_EHCI_BIG_ENDIAN_MMIO @@ -2102,6 +2103,9 @@ config SYS_SUPPORTS_HIGHMEM config SYS_SUPPORTS_SMARTMIPS bool +config SYS_SUPPORTS_MICROMIPS + bool + config ARCH_FLATMEM_ENABLE def_bool y depends on !NUMA && !CPU_LOONGSON2 diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h index f7d8f1509c4d..f924b87b5308 100644 --- a/arch/mips/include/asm/uasm.h +++ b/arch/mips/include/asm/uasm.h @@ -23,6 +23,7 @@ #endif #define _UASM_ISA_CLASSIC 0 +#define _UASM_ISA_MICROMIPS 1 #ifndef UASM_ISA #define UASM_ISA _UASM_ISA_CLASSIC @@ -31,6 +32,9 @@ #if (UASM_ISA == _UASM_ISA_CLASSIC) #define ISAOPC(op) uasm_i##op #define ISAFUNC(x) x +#elif (UASM_ISA == _UASM_ISA_MICROMIPS) +#define ISAOPC(op) MM_uasm_i##op +#define ISAFUNC(x) MM_##x #else #error Unsupported micro-assembler ISA!!! #endif diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile index 9e90c2182aa5..e87aae1f2e80 100644 --- a/arch/mips/mm/Makefile +++ b/arch/mips/mm/Makefile @@ -22,3 +22,5 @@ obj-$(CONFIG_IP22_CPU_SCACHE) += sc-ip22.o obj-$(CONFIG_R5000_CPU_SCACHE) += sc-r5k.o obj-$(CONFIG_RM7000_CPU_SCACHE) += sc-rm7k.o obj-$(CONFIG_MIPS_CPU_SCACHE) += sc-mips.o + +obj-$(CONFIG_SYS_SUPPORTS_MICROMIPS) += uasm-micromips.o diff --git a/arch/mips/mm/uasm-micromips.c b/arch/mips/mm/uasm-micromips.c new file mode 100644 index 000000000000..476d50c5fac1 --- /dev/null +++ b/arch/mips/mm/uasm-micromips.c @@ -0,0 +1,220 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * A small micro-assembler. It is intentionally kept simple, does only + * support a subset of instructions, and does not try to hide pipeline + * effects like branch delay slots. + * + * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer + * Copyright (C) 2005, 2007 Maciej W. Rozycki + * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) + * Copyright (C) 2012, 2013 MIPS Technologies, Inc. All rights reserved. + */ + +#include +#include +#include + +#include +#include +#include +#define UASM_ISA _UASM_ISA_MICROMIPS +#include + +#define RS_MASK 0x1f +#define RS_SH 16 +#define RT_MASK 0x1f +#define RT_SH 21 +#define SCIMM_MASK 0x3ff +#define SCIMM_SH 16 + +/* This macro sets the non-variable bits of an instruction. */ +#define M(a, b, c, d, e, f) \ + ((a) << OP_SH \ + | (b) << RT_SH \ + | (c) << RS_SH \ + | (d) << RD_SH \ + | (e) << RE_SH \ + | (f) << FUNC_SH) + +/* Define these when we are not the ISA the kernel is being compiled with. */ +#ifndef CONFIG_CPU_MICROMIPS +#define MM_uasm_i_b(buf, off) ISAOPC(_beq)(buf, 0, 0, off) +#define MM_uasm_i_beqz(buf, rs, off) ISAOPC(_beq)(buf, rs, 0, off) +#define MM_uasm_i_beqzl(buf, rs, off) ISAOPC(_beql)(buf, rs, 0, off) +#define MM_uasm_i_bnez(buf, rs, off) ISAOPC(_bne)(buf, rs, 0, off) +#endif + +#include "uasm.c" + +static struct insn insn_table_MM[] __uasminitdata = { + { insn_addu, M(mm_pool32a_op, 0, 0, 0, 0, mm_addu32_op), RT | RS | RD }, + { insn_addiu, M(mm_addiu32_op, 0, 0, 0, 0, 0), RT | RS | SIMM }, + { insn_and, M(mm_pool32a_op, 0, 0, 0, 0, mm_and_op), RT | RS | RD }, + { insn_andi, M(mm_andi32_op, 0, 0, 0, 0, 0), RT | RS | UIMM }, + { insn_beq, M(mm_beq32_op, 0, 0, 0, 0, 0), RS | RT | BIMM }, + { insn_beql, 0, 0 }, + { insn_bgez, M(mm_pool32i_op, mm_bgez_op, 0, 0, 0, 0), RS | BIMM }, + { insn_bgezl, 0, 0 }, + { insn_bltz, M(mm_pool32i_op, mm_bltz_op, 0, 0, 0, 0), RS | BIMM }, + { insn_bltzl, 0, 0 }, + { insn_bne, M(mm_bne32_op, 0, 0, 0, 0, 0), RT | RS | BIMM }, + { insn_cache, M(mm_pool32b_op, 0, 0, mm_cache_func, 0, 0), RT | RS | SIMM }, + { insn_daddu, 0, 0 }, + { insn_daddiu, 0, 0 }, + { insn_dmfc0, 0, 0 }, + { insn_dmtc0, 0, 0 }, + { insn_dsll, 0, 0 }, + { insn_dsll32, 0, 0 }, + { insn_dsra, 0, 0 }, + { insn_dsrl, 0, 0 }, + { insn_dsrl32, 0, 0 }, + { insn_drotr, 0, 0 }, + { insn_drotr32, 0, 0 }, + { insn_dsubu, 0, 0 }, + { insn_eret, M(mm_pool32a_op, 0, 0, 0, mm_eret_op, mm_pool32axf_op), 0 }, + { insn_ins, M(mm_pool32a_op, 0, 0, 0, 0, mm_ins_op), RT | RS | RD | RE }, + { insn_ext, M(mm_pool32a_op, 0, 0, 0, 0, mm_ext_op), RT | RS | RD | RE }, + { insn_j, M(mm_j32_op, 0, 0, 0, 0, 0), JIMM }, + { insn_jal, M(mm_jal32_op, 0, 0, 0, 0, 0), JIMM }, + { insn_jr, M(mm_pool32a_op, 0, 0, 0, mm_jalr_op, mm_pool32axf_op), RS }, + { insn_ld, 0, 0 }, + { insn_ll, M(mm_pool32c_op, 0, 0, (mm_ll_func << 1), 0, 0), RS | RT | SIMM }, + { insn_lld, 0, 0 }, + { insn_lui, M(mm_pool32i_op, mm_lui_op, 0, 0, 0, 0), RS | SIMM }, + { insn_lw, M(mm_lw32_op, 0, 0, 0, 0, 0), RT | RS | SIMM }, + { insn_mfc0, M(mm_pool32a_op, 0, 0, 0, mm_mfc0_op, mm_pool32axf_op), RT | RS | RD }, + { insn_mtc0, M(mm_pool32a_op, 0, 0, 0, mm_mtc0_op, mm_pool32axf_op), RT | RS | RD }, + { insn_or, M(mm_pool32a_op, 0, 0, 0, 0, mm_or32_op), RT | RS | RD }, + { insn_ori, M(mm_ori32_op, 0, 0, 0, 0, 0), RT | RS | UIMM }, + { insn_pref, M(mm_pool32c_op, 0, 0, (mm_pref_func << 1), 0, 0), RT | RS | SIMM }, + { insn_rfe, 0, 0 }, + { insn_sc, M(mm_pool32c_op, 0, 0, (mm_sc_func << 1), 0, 0), RT | RS | SIMM }, + { insn_scd, 0, 0 }, + { insn_sd, 0, 0 }, + { insn_sll, M(mm_pool32a_op, 0, 0, 0, 0, mm_sll32_op), RT | RS | RD }, + { insn_sra, M(mm_pool32a_op, 0, 0, 0, 0, mm_sra_op), RT | RS | RD }, + { insn_srl, M(mm_pool32a_op, 0, 0, 0, 0, mm_srl32_op), RT | RS | RD }, + { insn_rotr, M(mm_pool32a_op, 0, 0, 0, 0, mm_rotr_op), RT | RS | RD }, + { insn_subu, M(mm_pool32a_op, 0, 0, 0, 0, mm_subu32_op), RT | RS | RD }, + { insn_sw, M(mm_sw32_op, 0, 0, 0, 0, 0), RT | RS | SIMM }, + { insn_tlbp, M(mm_pool32a_op, 0, 0, 0, mm_tlbp_op, mm_pool32axf_op), 0 }, + { insn_tlbr, M(mm_pool32a_op, 0, 0, 0, mm_tlbr_op, mm_pool32axf_op), 0 }, + { insn_tlbwi, M(mm_pool32a_op, 0, 0, 0, mm_tlbwi_op, mm_pool32axf_op), 0 }, + { insn_tlbwr, M(mm_pool32a_op, 0, 0, 0, mm_tlbwr_op, mm_pool32axf_op), 0 }, + { insn_xor, M(mm_pool32a_op, 0, 0, 0, 0, mm_xor32_op), RT | RS | RD }, + { insn_xori, M(mm_xori32_op, 0, 0, 0, 0, 0), RT | RS | UIMM }, + { insn_dins, 0, 0 }, + { insn_dinsm, 0, 0 }, + { insn_syscall, M(mm_pool32a_op, 0, 0, 0, mm_syscall_op, mm_pool32axf_op), SCIMM}, + { insn_bbit0, 0, 0 }, + { insn_bbit1, 0, 0 }, + { insn_lwx, 0, 0 }, + { insn_ldx, 0, 0 }, + { insn_invalid, 0, 0 } +}; + +#undef M + +static inline __uasminit u32 build_bimm(s32 arg) +{ + WARN(arg > 0xffff || arg < -0x10000, + KERN_WARNING "Micro-assembler field overflow\n"); + + WARN(arg & 0x3, KERN_WARNING "Invalid micro-assembler branch target\n"); + + return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 1) & 0x7fff); +} + +static inline __uasminit u32 build_jimm(u32 arg) +{ + WARN(arg & ~(JIMM_MASK << 2), + KERN_WARNING "Micro-assembler field overflow\n"); + + return (arg >> 1) & JIMM_MASK; +} + +/* + * The order of opcode arguments is implicitly left to right, + * starting with RS and ending with FUNC or IMM. + */ +static void __uasminit build_insn(u32 **buf, enum opcode opc, ...) +{ + struct insn *ip = NULL; + unsigned int i; + va_list ap; + u32 op; + + for (i = 0; insn_table_MM[i].opcode != insn_invalid; i++) + if (insn_table_MM[i].opcode == opc) { + ip = &insn_table_MM[i]; + break; + } + + if (!ip || (opc == insn_daddiu && r4k_daddiu_bug())) + panic("Unsupported Micro-assembler instruction %d", opc); + + op = ip->match; + va_start(ap, opc); + if (ip->fields & RS) { + if (opc == insn_mfc0 || opc == insn_mtc0) + op |= build_rt(va_arg(ap, u32)); + else + op |= build_rs(va_arg(ap, u32)); + } + if (ip->fields & RT) { + if (opc == insn_mfc0 || opc == insn_mtc0) + op |= build_rs(va_arg(ap, u32)); + else + op |= build_rt(va_arg(ap, u32)); + } + if (ip->fields & RD) + op |= build_rd(va_arg(ap, u32)); + if (ip->fields & RE) + op |= build_re(va_arg(ap, u32)); + if (ip->fields & SIMM) + op |= build_simm(va_arg(ap, s32)); + if (ip->fields & UIMM) + op |= build_uimm(va_arg(ap, u32)); + if (ip->fields & BIMM) + op |= build_bimm(va_arg(ap, s32)); + if (ip->fields & JIMM) + op |= build_jimm(va_arg(ap, u32)); + if (ip->fields & FUNC) + op |= build_func(va_arg(ap, u32)); + if (ip->fields & SET) + op |= build_set(va_arg(ap, u32)); + if (ip->fields & SCIMM) + op |= build_scimm(va_arg(ap, u32)); + va_end(ap); + +#ifdef CONFIG_CPU_LITTLE_ENDIAN + **buf = ((op & 0xffff) << 16) | (op >> 16); +#else + **buf = op; +#endif + (*buf)++; +} + +static inline void __uasminit +__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab) +{ + long laddr = (long)lab->addr; + long raddr = (long)rel->addr; + + switch (rel->type) { + case R_MIPS_PC16: +#ifdef CONFIG_CPU_LITTLE_ENDIAN + *rel->addr |= (build_bimm(laddr - (raddr + 4)) << 16); +#else + *rel->addr |= build_bimm(laddr - (raddr + 4)); +#endif + break; + + default: + panic("Unsupported Micro-assembler relocation %d", + rel->type); + } +} -- cgit v1.2.3 From b6d92b4a6bdb880b39789c677b952c53a437028d Mon Sep 17 00:00:00 2001 From: "Steven J. Hill" Date: Mon, 25 Mar 2013 13:47:29 -0500 Subject: MIPS: Add option to disable software I/O coherency. Some MIPS controllers have hardware I/O coherency. This patch detects those and turns off software coherency. A new kernel command line option also allows the user to manually turn software coherency on or off. Signed-off-by: Steven J. Hill --- arch/mips/include/asm/dma-coherence.h | 15 ++++++ arch/mips/include/asm/dma-mapping.h | 1 + arch/mips/include/asm/mach-generic/dma-coherence.h | 5 +- arch/mips/mm/c-r4k.c | 24 +++------ arch/mips/mm/dma-default.c | 25 ++++++++- arch/mips/mti-malta/malta-setup.c | 63 ++++++++++++++++++++++ arch/mips/mti-sead3/sead3-setup.c | 3 -- 7 files changed, 112 insertions(+), 24 deletions(-) create mode 100644 arch/mips/include/asm/dma-coherence.h (limited to 'arch/mips/mm') diff --git a/arch/mips/include/asm/dma-coherence.h b/arch/mips/include/asm/dma-coherence.h new file mode 100644 index 000000000000..242cbb3ca582 --- /dev/null +++ b/arch/mips/include/asm/dma-coherence.h @@ -0,0 +1,15 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2006 Ralf Baechle + * + */ +#ifndef __ASM_DMA_COHERENCE_H +#define __ASM_DMA_COHERENCE_H + +extern int coherentio; +extern int hw_coherentio; + +#endif diff --git a/arch/mips/include/asm/dma-mapping.h b/arch/mips/include/asm/dma-mapping.h index f8fc74b6cb47..84238c574d5e 100644 --- a/arch/mips/include/asm/dma-mapping.h +++ b/arch/mips/include/asm/dma-mapping.h @@ -2,6 +2,7 @@ #define _ASM_DMA_MAPPING_H #include +#include #include #include diff --git a/arch/mips/include/asm/mach-generic/dma-coherence.h b/arch/mips/include/asm/mach-generic/dma-coherence.h index 9c95177f7a7e..fe23034aaf72 100644 --- a/arch/mips/include/asm/mach-generic/dma-coherence.h +++ b/arch/mips/include/asm/mach-generic/dma-coherence.h @@ -61,9 +61,8 @@ static inline int plat_device_is_coherent(struct device *dev) { #ifdef CONFIG_DMA_COHERENT return 1; -#endif -#ifdef CONFIG_DMA_NONCOHERENT - return 0; +#else + return coherentio; #endif } diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 2078915eacb9..f5943ab44987 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -33,6 +33,7 @@ #include #include /* for run_uncached() */ #include +#include /* * Special Variant of smp_call_function for use by cache functions: @@ -1377,20 +1378,6 @@ static void __cpuinit coherency_setup(void) } } -#if defined(CONFIG_DMA_NONCOHERENT) - -static int __cpuinitdata coherentio; - -static int __init setcoherentio(char *str) -{ - coherentio = 1; - - return 0; -} - -early_param("coherentio", setcoherentio); -#endif - static void __cpuinit r4k_cache_error_setup(void) { extern char __weak except_vec2_generic; @@ -1472,9 +1459,14 @@ void __cpuinit r4k_cache_init(void) build_clear_page(); build_copy_page(); -#if !defined(CONFIG_MIPS_CMP) + + /* + * We want to run CMP kernels on core with and without coherent + * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether + * or not to flush caches. + */ local_r4k___flush_cache_all(NULL); -#endif + coherency_setup(); board_cache_error_setup = r4k_cache_error_setup; } diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c index f9ef83829a52..caf92ecb37d6 100644 --- a/arch/mips/mm/dma-default.c +++ b/arch/mips/mm/dma-default.c @@ -22,6 +22,26 @@ #include +int coherentio = 0; /* User defined DMA coherency from command line. */ +EXPORT_SYMBOL_GPL(coherentio); +int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */ + +static int __init setcoherentio(char *str) +{ + coherentio = 1; + pr_info("Hardware DMA cache coherency (command line)\n"); + return 0; +} +early_param("coherentio", setcoherentio); + +static int __init setnocoherentio(char *str) +{ + coherentio = 0; + pr_info("Software DMA cache coherency (command line)\n"); + return 0; +} +early_param("nocoherentio", setnocoherentio); + static inline struct page *dma_addr_to_page(struct device *dev, dma_addr_t dma_addr) { @@ -115,7 +135,8 @@ static void *mips_dma_alloc_coherent(struct device *dev, size_t size, if (!plat_device_is_coherent(dev)) { dma_cache_wback_inv((unsigned long) ret, size); - ret = UNCAC_ADDR(ret); + if (!hw_coherentio) + ret = UNCAC_ADDR(ret); } } @@ -142,7 +163,7 @@ static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr, plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL); - if (!plat_device_is_coherent(dev)) + if (!plat_device_is_coherent(dev) && !hw_coherentio) addr = CAC_ADDR(addr); free_pages(addr, get_order(size)); diff --git a/arch/mips/mti-malta/malta-setup.c b/arch/mips/mti-malta/malta-setup.c index 200f64df2c9b..a85557171faa 100644 --- a/arch/mips/mti-malta/malta-setup.c +++ b/arch/mips/mti-malta/malta-setup.c @@ -32,6 +32,7 @@ #include #include #include +#include #ifdef CONFIG_VT #include #endif @@ -105,6 +106,66 @@ static void __init fd_activate(void) } #endif +static int __init plat_enable_iocoherency(void) +{ + int supported = 0; + if (mips_revision_sconid == MIPS_REVISION_SCON_BONITO) { + if (BONITO_PCICACHECTRL & BONITO_PCICACHECTRL_CPUCOH_PRES) { + BONITO_PCICACHECTRL |= BONITO_PCICACHECTRL_CPUCOH_EN; + pr_info("Enabled Bonito CPU coherency\n"); + supported = 1; + } + if (strstr(fw_getcmdline(), "iobcuncached")) { + BONITO_PCICACHECTRL &= ~BONITO_PCICACHECTRL_IOBCCOH_EN; + BONITO_PCIMEMBASECFG = BONITO_PCIMEMBASECFG & + ~(BONITO_PCIMEMBASECFG_MEMBASE0_CACHED | + BONITO_PCIMEMBASECFG_MEMBASE1_CACHED); + pr_info("Disabled Bonito IOBC coherency\n"); + } else { + BONITO_PCICACHECTRL |= BONITO_PCICACHECTRL_IOBCCOH_EN; + BONITO_PCIMEMBASECFG |= + (BONITO_PCIMEMBASECFG_MEMBASE0_CACHED | + BONITO_PCIMEMBASECFG_MEMBASE1_CACHED); + pr_info("Enabled Bonito IOBC coherency\n"); + } + } else if (gcmp_niocu() != 0) { + /* Nothing special needs to be done to enable coherency */ + pr_info("CMP IOCU detected\n"); + if ((*(unsigned int *)0xbf403000 & 0x81) != 0x81) { + pr_crit("IOCU OPERATION DISABLED BY SWITCH - DEFAULTING TO SW IO COHERENCY\n"); + return 0; + } + supported = 1; + } + hw_coherentio = supported; + return supported; +} + +static void __init plat_setup_iocoherency(void) +{ +#ifdef CONFIG_DMA_NONCOHERENT + /* + * Kernel has been configured with software coherency + * but we might choose to turn it off and use hardware + * coherency instead. + */ + if (plat_enable_iocoherency()) { + if (coherentio == 0) + pr_info("Hardware DMA cache coherency disabled\n"); + else + pr_info("Hardware DMA cache coherency enabled\n"); + } else { + if (coherentio == 1) + pr_info("Hardware DMA cache coherency unsupported, but enabled from command line!\n"); + else + pr_info("Software DMA cache coherency enabled\n"); + } +#else + if (!plat_enable_iocoherency()) + panic("Hardware DMA cache coherency not supported!"); +#endif +} + #ifdef CONFIG_BLK_DEV_IDE static void __init pci_clock_check(void) { @@ -207,6 +268,8 @@ void __init plat_mem_setup(void) if (mips_revision_sconid == MIPS_REVISION_SCON_BONITO) bonito_quirks_setup(); + plat_setup_iocoherency(); + #ifdef CONFIG_BLK_DEV_IDE pci_clock_check(); #endif diff --git a/arch/mips/mti-sead3/sead3-setup.c b/arch/mips/mti-sead3/sead3-setup.c index f012fd164cee..14633bc80049 100644 --- a/arch/mips/mti-sead3/sead3-setup.c +++ b/arch/mips/mti-sead3/sead3-setup.c @@ -13,9 +13,6 @@ #include #include -int coherentio; /* 0 => no DMA cache coherency (may be set by user) */ -int hw_coherentio; /* 0 => no HW DMA cache coherency (reflects real HW) */ - const char *get_system_type(void) { return "MIPS SEAD3"; -- cgit v1.2.3 From 8759934e2b6bdb3a08a81fc14a6588f3321719b1 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Sun, 17 Mar 2013 11:49:38 +0000 Subject: MIPS: Build uasm-generated code only once to avoid CPU Hotplug problem This and the next patch resolve memory corruption problems while CPU hotplug. Without these patches, memory corruption can triggered easily as below: On a quad-core MIPS platform, use "spawn" of UnixBench-5.1.3 (http:// code.google.com/p/byte-unixbench/) and a CPU hotplug script like this (hotplug.sh): while true; do echo 0 >/sys/devices/system/cpu/cpu1/online echo 0 >/sys/devices/system/cpu/cpu2/online echo 0 >/sys/devices/system/cpu/cpu3/online sleep 1 echo 1 >/sys/devices/system/cpu/cpu1/online echo 1 >/sys/devices/system/cpu/cpu2/online echo 1 >/sys/devices/system/cpu/cpu3/online sleep 1 done Run "hotplug.sh" and then run "spawn 10000", spawn will get segfault after a few minutes. This patch: Currently, clear_page()/copy_page() are generated by Micro-assembler dynamically. But they are unavailable until uasm_resolve_relocs() has finished because jump labels are illegal before that. Since these functions are shared by every CPU, we only call build_clear_page()/ build_copy_page() only once at boot time. Without this patch, programs will get random memory corruption (segmentation fault, bus error, etc.) while CPU Hotplug (e.g. one CPU is using clear_page() while another is generating it in cpu_cache_init()). For similar reasons we modify build_tlb_refill_handler()'s invocation. V2: 1, Rework the code to make CPU#0 can be online/offline. 2, Introduce cpu_has_local_ebase feature since some types of MIPS CPU need a per-CPU tlb_refill_handler(). Signed-off-by: Huacai Chen Signed-off-by: Hongbing Hu Acked-by: David Daney Patchwork: http://patchwork.linux-mips.org/patch/4994/ Acked-by: John Crispin --- arch/mips/include/asm/cpu-features.h | 3 +++ arch/mips/include/asm/mach-loongson/cpu-feature-overrides.h | 1 + arch/mips/mm/page.c | 10 ++++++++++ arch/mips/mm/tlbex.c | 10 ++++++++-- 4 files changed, 22 insertions(+), 2 deletions(-) (limited to 'arch/mips/mm') diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index 1a57e8b4d092..e5ec8fcd8afa 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -113,6 +113,9 @@ #ifndef cpu_has_pindexed_dcache #define cpu_has_pindexed_dcache (cpu_data[0].dcache.flags & MIPS_CACHE_PINDEX) #endif +#ifndef cpu_has_local_ebase +#define cpu_has_local_ebase 1 +#endif /* * I-Cache snoops remote store. This only matters on SMP. Some multiprocessors diff --git a/arch/mips/include/asm/mach-loongson/cpu-feature-overrides.h b/arch/mips/include/asm/mach-loongson/cpu-feature-overrides.h index 75fd8c0f986e..c0f3ef45c2c1 100644 --- a/arch/mips/include/asm/mach-loongson/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-loongson/cpu-feature-overrides.h @@ -57,5 +57,6 @@ #define cpu_has_vint 0 #define cpu_has_vtag_icache 0 #define cpu_has_watch 1 +#define cpu_has_local_ebase 0 #endif /* __ASM_MACH_LOONGSON_CPU_FEATURE_OVERRIDES_H */ diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c index a29fba55b53e..4eb8dcfaf1ce 100644 --- a/arch/mips/mm/page.c +++ b/arch/mips/mm/page.c @@ -247,6 +247,11 @@ void __cpuinit build_clear_page(void) struct uasm_label *l = labels; struct uasm_reloc *r = relocs; int i; + static atomic_t run_once = ATOMIC_INIT(0); + + if (atomic_xchg(&run_once, 1)) { + return; + } memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); @@ -389,6 +394,11 @@ void __cpuinit build_copy_page(void) struct uasm_label *l = labels; struct uasm_reloc *r = relocs; int i; + static atomic_t run_once = ATOMIC_INIT(0); + + if (atomic_xchg(&run_once, 1)) { + return; + } memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 820e6612d744..6bc28b4db31d 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -2162,8 +2162,11 @@ void __cpuinit build_tlb_refill_handler(void) case CPU_TX3922: case CPU_TX3927: #ifndef CONFIG_MIPS_PGD_C0_CONTEXT - build_r3000_tlb_refill_handler(); + if (cpu_has_local_ebase) + build_r3000_tlb_refill_handler(); if (!run_once) { + if (!cpu_has_local_ebase) + build_r3000_tlb_refill_handler(); build_r3000_tlb_load_handler(); build_r3000_tlb_store_handler(); build_r3000_tlb_modify_handler(); @@ -2192,9 +2195,12 @@ void __cpuinit build_tlb_refill_handler(void) build_r4000_tlb_load_handler(); build_r4000_tlb_store_handler(); build_r4000_tlb_modify_handler(); + if (!cpu_has_local_ebase) + build_r4000_tlb_refill_handler(); run_once++; } - build_r4000_tlb_refill_handler(); + if (cpu_has_local_ebase) + build_r4000_tlb_refill_handler(); } } -- cgit v1.2.3 From f2e3656d23b7e2465da22966feef6170e9b28b2b Mon Sep 17 00:00:00 2001 From: Sanjay Lal Date: Wed, 21 Nov 2012 18:34:10 -0800 Subject: MIPS: Export routines needed by the KVM module. Signed-off-by: Sanjay Lal Cc: kvm@vger.kernel.org Cc: linux-mips@linux-mips.org Signed-off-by: Ralf Baechle --- arch/mips/mm/c-r4k.c | 6 ++++-- arch/mips/mm/cache.c | 1 + arch/mips/mm/tlb-r4k.c | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) (limited to 'arch/mips/mm') diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index ecca559b8d7b..b5aedb2ea914 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -136,7 +136,8 @@ static void __cpuinit r4k_blast_dcache_page_indexed_setup(void) r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed; } -static void (* r4k_blast_dcache)(void); +void (* r4k_blast_dcache)(void); +EXPORT_SYMBOL(r4k_blast_dcache); static void __cpuinit r4k_blast_dcache_setup(void) { @@ -264,7 +265,8 @@ static void __cpuinit r4k_blast_icache_page_indexed_setup(void) r4k_blast_icache_page_indexed = blast_icache64_page_indexed; } -static void (* r4k_blast_icache)(void); +void (* r4k_blast_icache)(void); +EXPORT_SYMBOL(r4k_blast_icache); static void __cpuinit r4k_blast_icache_setup(void) { diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index 07cec4407b0c..5aeb3eb0b72f 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -48,6 +48,7 @@ void (*flush_icache_all)(void); EXPORT_SYMBOL_GPL(local_flush_data_cache_page); EXPORT_SYMBOL(flush_data_cache_page); +EXPORT_SYMBOL(flush_icache_all); #ifdef CONFIG_DMA_NONCOHERENT diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index 493131c81a29..c643de4c473a 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -94,6 +95,7 @@ void local_flush_tlb_all(void) FLUSH_ITLB; EXIT_CRITICAL(flags); } +EXPORT_SYMBOL(local_flush_tlb_all); /* All entries common to a mm share an asid. To effectively flush these entries, we just bump the asid. */ -- cgit v1.2.3 From d532f3d26716a39dfd4b88d687bd344fbe77e390 Mon Sep 17 00:00:00 2001 From: "Steven J. Hill" Date: Mon, 25 Mar 2013 11:58:57 -0500 Subject: MIPS: Allow ASID size to be determined at boot time. Original patch by Ralf Baechle and removed by Harold Koerfgen with commit f67e4ffc79905482c3b9b8c8dd65197bac7eb508. This allows for more generic kernels since the size of the ASID and corresponding masks can be determined at run-time. This patch is also required for the new Aptiv cores and has been tested on Malta and Malta Aptiv platforms. [ralf@linux-mips.org: Added relevant part of fix https://patchwork.linux-mips.org/patch/5213/] Signed-off-by: Steven J. Hill Signed-off-by: Ralf Baechle --- arch/mips/include/asm/mmu_context.h | 95 +++++++++++++++++++++++-------------- arch/mips/kernel/genex.S | 2 +- arch/mips/kernel/smtc.c | 10 ++-- arch/mips/kernel/traps.c | 6 ++- arch/mips/lib/dump_tlb.c | 5 +- arch/mips/lib/r3k_dump_tlb.c | 7 +-- arch/mips/mm/tlb-r3k.c | 20 ++++---- arch/mips/mm/tlb-r4k.c | 2 +- arch/mips/mm/tlb-r8k.c | 2 +- arch/mips/mm/tlbex.c | 49 +++++++++++++++++++ 10 files changed, 137 insertions(+), 61 deletions(-) (limited to 'arch/mips/mm') diff --git a/arch/mips/include/asm/mmu_context.h b/arch/mips/include/asm/mmu_context.h index e81d719efcd1..bab1980bbf0d 100644 --- a/arch/mips/include/asm/mmu_context.h +++ b/arch/mips/include/asm/mmu_context.h @@ -62,45 +62,68 @@ extern unsigned long pgd_current[]; TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir) #endif #endif /* CONFIG_MIPS_PGD_C0_CONTEXT*/ -#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) -#define ASID_INC 0x40 -#define ASID_MASK 0xfc0 - -#elif defined(CONFIG_CPU_R8000) - -#define ASID_INC 0x10 -#define ASID_MASK 0xff0 - -#elif defined(CONFIG_MIPS_MT_SMTC) - -#define ASID_INC 0x1 -extern unsigned long smtc_asid_mask; -#define ASID_MASK (smtc_asid_mask) -#define HW_ASID_MASK 0xff -/* End SMTC/34K debug hack */ -#else /* FIXME: not correct for R6000 */ - -#define ASID_INC 0x1 -#define ASID_MASK 0xff +#define ASID_INC(asid) \ +({ \ + unsigned long __asid = asid; \ + __asm__("1:\taddiu\t%0,1\t\t\t\t# patched\n\t" \ + ".section\t__asid_inc,\"a\"\n\t" \ + ".word\t1b\n\t" \ + ".previous" \ + :"=r" (__asid) \ + :"0" (__asid)); \ + __asid; \ +}) +#define ASID_MASK(asid) \ +({ \ + unsigned long __asid = asid; \ + __asm__("1:\tandi\t%0,%1,0xfc0\t\t\t# patched\n\t" \ + ".section\t__asid_mask,\"a\"\n\t" \ + ".word\t1b\n\t" \ + ".previous" \ + :"=r" (__asid) \ + :"r" (__asid)); \ + __asid; \ +}) +#define ASID_VERSION_MASK \ +({ \ + unsigned long __asid; \ + __asm__("1:\taddiu\t%0,$0,0xff00\t\t\t\t# patched\n\t" \ + ".section\t__asid_version_mask,\"a\"\n\t" \ + ".word\t1b\n\t" \ + ".previous" \ + :"=r" (__asid)); \ + __asid; \ +}) +#define ASID_FIRST_VERSION \ +({ \ + unsigned long __asid = asid; \ + __asm__("1:\tli\t%0,0x100\t\t\t\t# patched\n\t" \ + ".section\t__asid_first_version,\"a\"\n\t" \ + ".word\t1b\n\t" \ + ".previous" \ + :"=r" (__asid)); \ + __asid; \ +}) + +#define ASID_FIRST_VERSION_R3000 0x1000 +#define ASID_FIRST_VERSION_R4000 0x100 +#define ASID_FIRST_VERSION_R8000 0x1000 +#define ASID_FIRST_VERSION_RM9000 0x1000 +#ifdef CONFIG_MIPS_MT_SMTC +#define SMTC_HW_ASID_MASK 0xff +extern unsigned int smtc_asid_mask; #endif #define cpu_context(cpu, mm) ((mm)->context.asid[cpu]) -#define cpu_asid(cpu, mm) (cpu_context((cpu), (mm)) & ASID_MASK) +#define cpu_asid(cpu, mm) ASID_MASK(cpu_context((cpu), (mm))) #define asid_cache(cpu) (cpu_data[cpu].asid_cache) static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) { } -/* - * All unused by hardware upper bits will be considered - * as a software asid extension. - */ -#define ASID_VERSION_MASK ((unsigned long)~(ASID_MASK|(ASID_MASK-1))) -#define ASID_FIRST_VERSION ((unsigned long)(~ASID_VERSION_MASK) + 1) - #ifndef CONFIG_MIPS_MT_SMTC /* Normal, classic MIPS get_new_mmu_context */ static inline void @@ -108,7 +131,7 @@ get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) { unsigned long asid = asid_cache(cpu); - if (! ((asid += ASID_INC) & ASID_MASK) ) { + if (!ASID_MASK((asid = ASID_INC(asid)))) { if (cpu_has_vtag_icache) flush_icache_all(); local_flush_tlb_all(); /* start new asid cycle */ @@ -166,7 +189,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, * free up the ASID value for use and flush any old * instances of it from the TLB. */ - oldasid = (read_c0_entryhi() & ASID_MASK); + oldasid = ASID_MASK(read_c0_entryhi()); if(smtc_live_asid[mytlb][oldasid]) { smtc_live_asid[mytlb][oldasid] &= ~(0x1 << cpu); if(smtc_live_asid[mytlb][oldasid] == 0) @@ -177,7 +200,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, * having ASID_MASK smaller than the hardware maximum, * make sure no "soft" bits become "hard"... */ - write_c0_entryhi((read_c0_entryhi() & ~HW_ASID_MASK) | + write_c0_entryhi((read_c0_entryhi() & ~SMTC_HW_ASID_MASK) | cpu_asid(cpu, next)); ehb(); /* Make sure it propagates to TCStatus */ evpe(mtflags); @@ -230,15 +253,15 @@ activate_mm(struct mm_struct *prev, struct mm_struct *next) #ifdef CONFIG_MIPS_MT_SMTC /* See comments for similar code above */ mtflags = dvpe(); - oldasid = read_c0_entryhi() & ASID_MASK; + oldasid = ASID_MASK(read_c0_entryhi()); if(smtc_live_asid[mytlb][oldasid]) { smtc_live_asid[mytlb][oldasid] &= ~(0x1 << cpu); if(smtc_live_asid[mytlb][oldasid] == 0) smtc_flush_tlb_asid(oldasid); } /* See comments for similar code above */ - write_c0_entryhi((read_c0_entryhi() & ~HW_ASID_MASK) | - cpu_asid(cpu, next)); + write_c0_entryhi((read_c0_entryhi() & ~SMTC_HW_ASID_MASK) | + cpu_asid(cpu, next)); ehb(); /* Make sure it propagates to TCStatus */ evpe(mtflags); #else @@ -275,14 +298,14 @@ drop_mmu_context(struct mm_struct *mm, unsigned cpu) #ifdef CONFIG_MIPS_MT_SMTC /* See comments for similar code above */ prevvpe = dvpe(); - oldasid = (read_c0_entryhi() & ASID_MASK); + oldasid = ASID_MASK(read_c0_entryhi()); if (smtc_live_asid[mytlb][oldasid]) { smtc_live_asid[mytlb][oldasid] &= ~(0x1 << cpu); if(smtc_live_asid[mytlb][oldasid] == 0) smtc_flush_tlb_asid(oldasid); } /* See comments for similar code above */ - write_c0_entryhi((read_c0_entryhi() & ~HW_ASID_MASK) + write_c0_entryhi((read_c0_entryhi() & ~SMTC_HW_ASID_MASK) | cpu_asid(cpu, mm)); ehb(); /* Make sure it propagates to TCStatus */ evpe(prevvpe); diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S index ecb347ce1b3d..5360b1db337d 100644 --- a/arch/mips/kernel/genex.S +++ b/arch/mips/kernel/genex.S @@ -480,7 +480,7 @@ NESTED(nmi_handler, PT_SIZE, sp) .set noreorder /* check if TLB contains a entry for EPC */ MFC0 k1, CP0_ENTRYHI - andi k1, 0xff /* ASID_MASK */ + andi k1, 0xff /* ASID_MASK patched at run-time!! */ MFC0 k0, CP0_EPC PTR_SRL k0, _PAGE_SHIFT + 1 PTR_SLL k0, _PAGE_SHIFT + 1 diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c index 7186222dc5bb..31d22f3121c9 100644 --- a/arch/mips/kernel/smtc.c +++ b/arch/mips/kernel/smtc.c @@ -111,7 +111,7 @@ static int vpe0limit; static int ipibuffers; static int nostlb; static int asidmask; -unsigned long smtc_asid_mask = 0xff; +unsigned int smtc_asid_mask = 0xff; static int __init vpe0tcs(char *str) { @@ -1395,7 +1395,7 @@ void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) asid = asid_cache(cpu); do { - if (!((asid += ASID_INC) & ASID_MASK) ) { + if (!ASID_MASK(ASID_INC(asid))) { if (cpu_has_vtag_icache) flush_icache_all(); /* Traverse all online CPUs (hack requires contiguous range) */ @@ -1414,7 +1414,7 @@ void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) mips_ihb(); } tcstat = read_tc_c0_tcstatus(); - smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i); + smtc_live_asid[tlb][ASID_MASK(tcstat)] |= (asiduse)(0x1 << i); if (!prevhalt) write_tc_c0_tchalt(0); } @@ -1423,7 +1423,7 @@ void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) asid = ASID_FIRST_VERSION; local_flush_tlb_all(); /* start new asid cycle */ } - } while (smtc_live_asid[tlb][(asid & ASID_MASK)]); + } while (smtc_live_asid[tlb][ASID_MASK(asid)]); /* * SMTC shares the TLB within VPEs and possibly across all VPEs. @@ -1461,7 +1461,7 @@ void smtc_flush_tlb_asid(unsigned long asid) tlb_read(); ehb(); ehi = read_c0_entryhi(); - if ((ehi & ASID_MASK) == asid) { + if (ASID_MASK(ehi) == asid) { /* * Invalidate only entries with specified ASID, * makiing sure all entries differ. diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index c3abb88170fc..4b6b607b0179 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -1547,6 +1547,7 @@ void __cpuinit per_cpu_trap_init(bool is_boot_cpu) unsigned int cpu = smp_processor_id(); unsigned int status_set = ST0_CU0; unsigned int hwrena = cpu_hwrena_impl_bits; + unsigned long asid = 0; #ifdef CONFIG_MIPS_MT_SMTC int secondaryTC = 0; int bootTC = (cpu == 0); @@ -1630,8 +1631,9 @@ void __cpuinit per_cpu_trap_init(bool is_boot_cpu) } #endif /* CONFIG_MIPS_MT_SMTC */ - if (!cpu_data[cpu].asid_cache) - cpu_data[cpu].asid_cache = ASID_FIRST_VERSION; + asid = ASID_FIRST_VERSION; + cpu_data[cpu].asid_cache = asid; + TLBMISS_HANDLER_SETUP(); atomic_inc(&init_mm.mm_count); current->active_mm = &init_mm; diff --git a/arch/mips/lib/dump_tlb.c b/arch/mips/lib/dump_tlb.c index 32b9f21bfd85..8a12d00908e0 100644 --- a/arch/mips/lib/dump_tlb.c +++ b/arch/mips/lib/dump_tlb.c @@ -11,6 +11,7 @@ #include #include #include +#include static inline const char *msk2str(unsigned int mask) { @@ -55,7 +56,7 @@ static void dump_tlb(int first, int last) s_pagemask = read_c0_pagemask(); s_entryhi = read_c0_entryhi(); s_index = read_c0_index(); - asid = s_entryhi & 0xff; + asid = ASID_MASK(s_entryhi); for (i = first; i <= last; i++) { write_c0_index(i); @@ -85,7 +86,7 @@ static void dump_tlb(int first, int last) printk("va=%0*lx asid=%02lx\n", width, (entryhi & ~0x1fffUL), - entryhi & 0xff); + ASID_MASK(entryhi)); printk("\t[pa=%0*llx c=%d d=%d v=%d g=%d] ", width, (entrylo0 << 6) & PAGE_MASK, c0, diff --git a/arch/mips/lib/r3k_dump_tlb.c b/arch/mips/lib/r3k_dump_tlb.c index 91615c2ef0cf..8327698b9937 100644 --- a/arch/mips/lib/r3k_dump_tlb.c +++ b/arch/mips/lib/r3k_dump_tlb.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -21,7 +22,7 @@ static void dump_tlb(int first, int last) unsigned int asid; unsigned long entryhi, entrylo0; - asid = read_c0_entryhi() & 0xfc0; + asid = ASID_MASK(read_c0_entryhi()); for (i = first; i <= last; i++) { write_c0_index(i<<8); @@ -35,7 +36,7 @@ static void dump_tlb(int first, int last) /* Unused entries have a virtual address of KSEG0. */ if ((entryhi & 0xffffe000) != 0x80000000 - && (entryhi & 0xfc0) == asid) { + && (ASID_MASK(entryhi) == asid)) { /* * Only print entries in use */ @@ -44,7 +45,7 @@ static void dump_tlb(int first, int last) printk("va=%08lx asid=%08lx" " [pa=%06lx n=%d d=%d v=%d g=%d]", (entryhi & 0xffffe000), - entryhi & 0xfc0, + ASID_MASK(entryhi), entrylo0 & PAGE_MASK, (entrylo0 & (1 << 11)) ? 1 : 0, (entrylo0 & (1 << 10)) ? 1 : 0, diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c index a63d1ed0827f..4a13c150f31b 100644 --- a/arch/mips/mm/tlb-r3k.c +++ b/arch/mips/mm/tlb-r3k.c @@ -51,7 +51,7 @@ void local_flush_tlb_all(void) #endif local_irq_save(flags); - old_ctx = read_c0_entryhi() & ASID_MASK; + old_ctx = ASID_MASK(read_c0_entryhi()); write_c0_entrylo0(0); entry = r3k_have_wired_reg ? read_c0_wired() : 8; for (; entry < current_cpu_data.tlbsize; entry++) { @@ -87,13 +87,13 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, #ifdef DEBUG_TLB printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", - cpu_context(cpu, mm) & ASID_MASK, start, end); + ASID_MASK(cpu_context(cpu, mm)), start, end); #endif local_irq_save(flags); size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; if (size <= current_cpu_data.tlbsize) { - int oldpid = read_c0_entryhi() & ASID_MASK; - int newpid = cpu_context(cpu, mm) & ASID_MASK; + int oldpid = ASID_MASK(read_c0_entryhi()); + int newpid = ASID_MASK(cpu_context(cpu, mm)); start &= PAGE_MASK; end += PAGE_SIZE - 1; @@ -166,10 +166,10 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) #ifdef DEBUG_TLB printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page); #endif - newpid = cpu_context(cpu, vma->vm_mm) & ASID_MASK; + newpid = ASID_MASK(cpu_context(cpu, vma->vm_mm)); page &= PAGE_MASK; local_irq_save(flags); - oldpid = read_c0_entryhi() & ASID_MASK; + oldpid = ASID_MASK(read_c0_entryhi()); write_c0_entryhi(page | newpid); BARRIER; tlb_probe(); @@ -197,10 +197,10 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte) if (current->active_mm != vma->vm_mm) return; - pid = read_c0_entryhi() & ASID_MASK; + pid = ASID_MASK(read_c0_entryhi()); #ifdef DEBUG_TLB - if ((pid != (cpu_context(cpu, vma->vm_mm) & ASID_MASK)) || (cpu_context(cpu, vma->vm_mm) == 0)) { + if ((pid != ASID_MASK(cpu_context(cpu, vma->vm_mm))) || (cpu_context(cpu, vma->vm_mm) == 0)) { printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n", (cpu_context(cpu, vma->vm_mm)), pid); } @@ -241,7 +241,7 @@ void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, local_irq_save(flags); /* Save old context and create impossible VPN2 value */ - old_ctx = read_c0_entryhi() & ASID_MASK; + old_ctx = ASID_MASK(read_c0_entryhi()); old_pagemask = read_c0_pagemask(); w = read_c0_wired(); write_c0_wired(w + 1); @@ -264,7 +264,7 @@ void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, #endif local_irq_save(flags); - old_ctx = read_c0_entryhi() & ASID_MASK; + old_ctx = ASID_MASK(read_c0_entryhi()); write_c0_entrylo0(entrylo0); write_c0_entryhi(entryhi); write_c0_index(wired); diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c index 493131c81a29..3d03a2ccebaa 100644 --- a/arch/mips/mm/tlb-r4k.c +++ b/arch/mips/mm/tlb-r4k.c @@ -285,7 +285,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) ENTER_CRITICAL(flags); - pid = read_c0_entryhi() & ASID_MASK; + pid = ASID_MASK(read_c0_entryhi()); address &= (PAGE_MASK << 1); write_c0_entryhi(address | pid); pgdp = pgd_offset(vma->vm_mm, address); diff --git a/arch/mips/mm/tlb-r8k.c b/arch/mips/mm/tlb-r8k.c index 91c2499f806a..122f9207f49e 100644 --- a/arch/mips/mm/tlb-r8k.c +++ b/arch/mips/mm/tlb-r8k.c @@ -195,7 +195,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) if (current->active_mm != vma->vm_mm) return; - pid = read_c0_entryhi() & ASID_MASK; + pid = ASID_MASK(read_c0_entryhi()); local_irq_save(flags); address &= PAGE_MASK; diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 820e6612d744..e2a9e3687c45 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -305,6 +306,48 @@ static struct uasm_reloc relocs[128] __cpuinitdata; static int check_for_high_segbits __cpuinitdata; #endif +static void __cpuinit insn_fixup(unsigned int **start, unsigned int **stop, + unsigned int i_const) +{ + unsigned int **p, *ip; + + for (p = start; p < stop; p++) { + ip = *p; + *ip = (*ip & 0xffff0000) | i_const; + } + local_flush_icache_range((unsigned long)*p, (unsigned long)((*p) + 1)); +} + +#define asid_insn_fixup(section, const) \ +do { \ + extern unsigned int *__start_ ## section; \ + extern unsigned int *__stop_ ## section; \ + insn_fixup(&__start_ ## section, &__stop_ ## section, const); \ +} while(0) + +/* + * Caller is assumed to flush the caches before the first context switch. + */ +static void __cpuinit setup_asid(unsigned int inc, unsigned int mask, + unsigned int version_mask, + unsigned int first_version) +{ + extern asmlinkage void handle_ri_rdhwr_vivt(void); + unsigned long *vivt_exc; + + asid_insn_fixup(__asid_inc, inc); + asid_insn_fixup(__asid_mask, mask); + asid_insn_fixup(__asid_version_mask, version_mask); + asid_insn_fixup(__asid_first_version, first_version); + + /* Patch up the 'handle_ri_rdhwr_vivt' handler. */ + vivt_exc = (unsigned long *) &handle_ri_rdhwr_vivt; + vivt_exc++; + *vivt_exc = (*vivt_exc & ~mask) | mask; + + current_cpu_data.asid_cache = first_version; +} + static int check_for_high_segbits __cpuinitdata; static unsigned int kscratch_used_mask __cpuinitdata; @@ -2162,6 +2205,7 @@ void __cpuinit build_tlb_refill_handler(void) case CPU_TX3922: case CPU_TX3927: #ifndef CONFIG_MIPS_PGD_C0_CONTEXT + setup_asid(0x40, 0xfc0, 0xf000, ASID_FIRST_VERSION_R3000); build_r3000_tlb_refill_handler(); if (!run_once) { build_r3000_tlb_load_handler(); @@ -2184,6 +2228,11 @@ void __cpuinit build_tlb_refill_handler(void) break; default: +#ifndef CONFIG_MIPS_MT_SMTC + setup_asid(0x1, 0xff, 0xff00, ASID_FIRST_VERSION_R4000); +#else + setup_asid(0x1, smtc_asid_mask, 0xff00, ASID_FIRST_VERSION_R4000); +#endif if (!run_once) { scratch_reg = allocate_kscratch(); #ifdef CONFIG_MIPS_PGD_C0_CONTEXT -- cgit v1.2.3 From f6b06d9361a008afb93b97fb3683a6e92d69d0f4 Mon Sep 17 00:00:00 2001 From: "Steven J. Hill" Date: Mon, 25 Mar 2013 12:01:00 -0500 Subject: MIPS: microMIPS: Support dynamic ASID sizing. Changes for pure microMIPS cores to dynamically determine the ASID size at boot time. Includes bug fix https://patchwork.linux-mips.org/patch/5230/ Signed-off-by: Steven J. Hill Signed-off-by: Jonas Gorski --- arch/mips/mm/tlbex.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) (limited to 'arch/mips/mm') diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index e2a9e3687c45..fb4ca9984ed1 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -309,13 +309,32 @@ static int check_for_high_segbits __cpuinitdata; static void __cpuinit insn_fixup(unsigned int **start, unsigned int **stop, unsigned int i_const) { - unsigned int **p, *ip; + unsigned int **p; for (p = start; p < stop; p++) { +#ifndef CONFIG_CPU_MICROMIPS + unsigned int *ip; + ip = *p; *ip = (*ip & 0xffff0000) | i_const; +#else + unsigned short *ip; + + ip = ((unsigned short *)((unsigned int)*p - 1)); + if ((*ip & 0xf000) == 0x4000) { + *ip &= 0xfff1; + *ip |= (i_const << 1); + } else if ((*ip & 0xf000) == 0x6000) { + *ip &= 0xfff1; + *ip |= ((i_const >> 2) << 1); + } else { + ip++; + *ip = i_const; + } +#endif + local_flush_icache_range((unsigned long)ip, + (unsigned long)ip + sizeof(*ip)); } - local_flush_icache_range((unsigned long)*p, (unsigned long)((*p) + 1)); } #define asid_insn_fixup(section, const) \ @@ -335,6 +354,14 @@ static void __cpuinit setup_asid(unsigned int inc, unsigned int mask, extern asmlinkage void handle_ri_rdhwr_vivt(void); unsigned long *vivt_exc; +#ifdef CONFIG_CPU_MICROMIPS + /* + * Worst case optimised microMIPS addiu instructions support + * only a 3-bit immediate value. + */ + if(inc > 7) + panic("Invalid ASID increment value!"); +#endif asid_insn_fixup(__asid_inc, inc); asid_insn_fixup(__asid_mask, mask); asid_insn_fixup(__asid_version_mask, version_mask); @@ -342,6 +369,9 @@ static void __cpuinit setup_asid(unsigned int inc, unsigned int mask, /* Patch up the 'handle_ri_rdhwr_vivt' handler. */ vivt_exc = (unsigned long *) &handle_ri_rdhwr_vivt; +#ifdef CONFIG_CPU_MICROMIPS + vivt_exc = (unsigned long *)((unsigned long) vivt_exc - 1); +#endif vivt_exc++; *vivt_exc = (*vivt_exc & ~mask) | mask; -- cgit v1.2.3 From 8fe4bb98e42945ddf2c0d47cec647ef76909e812 Mon Sep 17 00:00:00 2001 From: "Steven J. Hill" Date: Mon, 25 Mar 2013 12:07:40 -0500 Subject: MIPS: microMIPS: Fix incorrect mask for jump immediate. Jump or branch target addresses have the first bit set. The original mask did not take this into account and will cause a field overflow warning for the target address when a jump immediate instruction is built. Signed-off-by: Steven J. Hill --- arch/mips/mm/uasm-micromips.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/mips/mm') diff --git a/arch/mips/mm/uasm-micromips.c b/arch/mips/mm/uasm-micromips.c index 476d50c5fac1..162ee6d62788 100644 --- a/arch/mips/mm/uasm-micromips.c +++ b/arch/mips/mm/uasm-micromips.c @@ -130,7 +130,8 @@ static inline __uasminit u32 build_bimm(s32 arg) static inline __uasminit u32 build_jimm(u32 arg) { - WARN(arg & ~(JIMM_MASK << 2), + + WARN(arg & ~((JIMM_MASK << 2) | 1), KERN_WARNING "Micro-assembler field overflow\n"); return (arg >> 1) & JIMM_MASK; -- cgit v1.2.3 From cf6d905828c2c75ebe8c818901e71e09ffe6f629 Mon Sep 17 00:00:00 2001 From: "Steven J. Hill" Date: Mon, 25 Mar 2013 12:03:41 -0500 Subject: MIPS: microMIPS: Fix macro naming in micro-assembler. The macros did not properly take into account the ISA that the kernel was being compiled with. A classic MIPS kernel will have the standard 'uasm_i_##op' macro functions with 'MM_uasm_i_##op' macro functions for the microMIPS version. A pure microMIPS kernel will have the standard macros with 'CL_uasm_i_##op' macro functions for the classic version. Signed-off-by: Steven J. Hill --- arch/mips/include/asm/uasm.h | 18 ++++++++++++++++-- arch/mips/mm/uasm-mips.c | 9 +++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) (limited to 'arch/mips/mm') diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h index f924b87b5308..370d967725c2 100644 --- a/arch/mips/include/asm/uasm.h +++ b/arch/mips/include/asm/uasm.h @@ -26,15 +26,29 @@ #define _UASM_ISA_MICROMIPS 1 #ifndef UASM_ISA +#ifdef CONFIG_CPU_MICROMIPS +#define UASM_ISA _UASM_ISA_MICROMIPS +#else #define UASM_ISA _UASM_ISA_CLASSIC #endif +#endif #if (UASM_ISA == _UASM_ISA_CLASSIC) +#ifdef CONFIG_CPU_MICROMIPS +#define ISAOPC(op) CL_uasm_i##op +#define ISAFUNC(x) CL_##x +#else #define ISAOPC(op) uasm_i##op #define ISAFUNC(x) x +#endif #elif (UASM_ISA == _UASM_ISA_MICROMIPS) +#ifdef CONFIG_CPU_MICROMIPS +#define ISAOPC(op) uasm_i##op +#define ISAFUNC(x) x +#else #define ISAOPC(op) MM_uasm_i##op #define ISAFUNC(x) MM_##x +#endif #else #error Unsupported micro-assembler ISA!!! #endif @@ -160,9 +174,9 @@ void ISAFUNC(UASM_i_LA_mostly)(u32 **buf, unsigned int rs, long addr); void ISAFUNC(UASM_i_LA)(u32 **buf, unsigned int rs, long addr); #define UASM_L_LA(lb) \ -static inline void __uasminit uasm_l##lb(struct uasm_label **lab, u32 *addr) \ +static inline void __uasminit ISAFUNC(uasm_l##lb)(struct uasm_label **lab, u32 *addr) \ { \ - uasm_build_label(lab, addr, label##lb); \ + ISAFUNC(uasm_build_label)(lab, addr, label##lb); \ } /* convenience macros for instructions */ diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c index e78e74dc3aea..5fcdd8fe3e83 100644 --- a/arch/mips/mm/uasm-mips.c +++ b/arch/mips/mm/uasm-mips.c @@ -20,6 +20,7 @@ #include #include #include +#define UASM_ISA _UASM_ISA_CLASSIC #include #define RS_MASK 0x1f @@ -38,6 +39,14 @@ | (e) << RE_SH \ | (f) << FUNC_SH) +/* Define these when we are not the ISA the kernel is being compiled with. */ +#ifdef CONFIG_CPU_MICROMIPS +#define CL_uasm_i_b(buf, off) ISAOPC(_beq)(buf, 0, 0, off) +#define CL_uasm_i_beqz(buf, rs, off) ISAOPC(_beq)(buf, rs, 0, off) +#define CL_uasm_i_beqzl(buf, rs, off) ISAOPC(_beql)(buf, rs, 0, off) +#define CL_uasm_i_bnez(buf, rs, off) ISAOPC(_bne)(buf, rs, 0, off) +#endif + #include "uasm.c" static struct insn insn_table[] __uasminitdata = { -- cgit v1.2.3 From 2a0b24f56c2492b932f1aed617ae80fb23500d21 Mon Sep 17 00:00:00 2001 From: "Steven J. Hill" Date: Mon, 25 Mar 2013 12:15:55 -0500 Subject: MIPS: microMIPS: Add support for exception handling. All exceptions must be taken in microMIPS mode, never in classic MIPS mode or the kernel falls apart. A few NOP instructions are used to maintain the correct alignment of microMIPS versions of the exception vectors. Signed-off-by: Steven J. Hill --- arch/mips/include/asm/mipsregs.h | 1 + arch/mips/include/asm/stackframe.h | 12 +- arch/mips/kernel/cpu-probe.c | 3 + arch/mips/kernel/genex.S | 73 +++++++--- arch/mips/kernel/scall32-o32.S | 9 ++ arch/mips/kernel/smtc-asm.S | 3 + arch/mips/kernel/traps.c | 290 ++++++++++++++++++++++++++----------- arch/mips/mm/tlbex.c | 21 +++ arch/mips/mti-sead3/sead3-init.c | 48 ++++++ 9 files changed, 353 insertions(+), 107 deletions(-) (limited to 'arch/mips/mm') diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index f64e17fa3e96..87e6207b05e4 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -596,6 +596,7 @@ #define MIPS_CONF3_RXI (_ULCAST_(1) << 12) #define MIPS_CONF3_ULRI (_ULCAST_(1) << 13) #define MIPS_CONF3_ISA (_ULCAST_(3) << 14) +#define MIPS_CONF3_ISA_OE (_ULCAST_(3) << 16) #define MIPS_CONF3_VZ (_ULCAST_(1) << 23) #define MIPS_CONF4_MMUSIZEEXT (_ULCAST_(255) << 0) diff --git a/arch/mips/include/asm/stackframe.h b/arch/mips/include/asm/stackframe.h index c99384018161..a89d1b10d027 100644 --- a/arch/mips/include/asm/stackframe.h +++ b/arch/mips/include/asm/stackframe.h @@ -139,7 +139,7 @@ 1: move ra, k0 li k0, 3 mtc0 k0, $22 -#endif /* CONFIG_CPU_LOONGSON2F */ +#endif /* CONFIG_CPU_JUMP_WORKAROUNDS */ #if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32) lui k1, %hi(kernelsp) #else @@ -189,6 +189,7 @@ LONG_S $0, PT_R0(sp) mfc0 v1, CP0_STATUS LONG_S $2, PT_R2(sp) + LONG_S v1, PT_STATUS(sp) #ifdef CONFIG_MIPS_MT_SMTC /* * Ideally, these instructions would be shuffled in @@ -200,21 +201,20 @@ LONG_S k0, PT_TCSTATUS(sp) #endif /* CONFIG_MIPS_MT_SMTC */ LONG_S $4, PT_R4(sp) - LONG_S $5, PT_R5(sp) - LONG_S v1, PT_STATUS(sp) mfc0 v1, CP0_CAUSE - LONG_S $6, PT_R6(sp) - LONG_S $7, PT_R7(sp) + LONG_S $5, PT_R5(sp) LONG_S v1, PT_CAUSE(sp) + LONG_S $6, PT_R6(sp) MFC0 v1, CP0_EPC + LONG_S $7, PT_R7(sp) #ifdef CONFIG_64BIT LONG_S $8, PT_R8(sp) LONG_S $9, PT_R9(sp) #endif + LONG_S v1, PT_EPC(sp) LONG_S $25, PT_R25(sp) LONG_S $28, PT_R28(sp) LONG_S $31, PT_R31(sp) - LONG_S v1, PT_EPC(sp) ori $28, sp, _THREAD_MASK xori $28, _THREAD_MASK #ifdef CONFIG_CPU_CAVIUM_OCTEON diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index 5fe66a0c3224..4bbffdb9024f 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -470,6 +470,9 @@ static inline unsigned int decode_config3(struct cpuinfo_mips *c) c->options |= MIPS_CPU_ULRI; if (config3 & MIPS_CONF3_ISA) c->options |= MIPS_CPU_MICROMIPS; +#ifdef CONFIG_CPU_MICROMIPS + write_c0_config3(read_c0_config3() | MIPS_CONF3_ISA_OE); +#endif if (config3 & MIPS_CONF3_VZ) c->ases |= MIPS_ASE_VZ; diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S index 5360b1db337d..5c2ba9f08a80 100644 --- a/arch/mips/kernel/genex.S +++ b/arch/mips/kernel/genex.S @@ -5,8 +5,8 @@ * * Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle * Copyright (C) 1999, 2000 Silicon Graphics, Inc. - * Copyright (C) 2001 MIPS Technologies, Inc. * Copyright (C) 2002, 2007 Maciej W. Rozycki + * Copyright (C) 2001, 2012 MIPS Technologies, Inc. All rights reserved. */ #include @@ -21,8 +21,10 @@ #include #include +#ifdef CONFIG_MIPS_MT_SMTC #define PANIC_PIC(msg) \ - .set push; \ + .set push; \ + .set nomicromips; \ .set reorder; \ PTR_LA a0,8f; \ .set noat; \ @@ -31,17 +33,10 @@ 9: b 9b; \ .set pop; \ TEXT(msg) +#endif __INIT -NESTED(except_vec0_generic, 0, sp) - PANIC_PIC("Exception vector 0 called") - END(except_vec0_generic) - -NESTED(except_vec1_generic, 0, sp) - PANIC_PIC("Exception vector 1 called") - END(except_vec1_generic) - /* * General exception vector for all other CPUs. * @@ -138,12 +133,19 @@ LEAF(r4k_wait) nop nop nop +#ifdef CONFIG_CPU_MICROMIPS + nop + nop + nop + nop +#endif .set mips3 wait /* end of rollback region (the region size must be power of two) */ - .set pop 1: jr ra + nop + .set pop END(r4k_wait) .macro BUILD_ROLLBACK_PROLOGUE handler @@ -201,7 +203,11 @@ NESTED(handle_int, PT_SIZE, sp) LONG_L s0, TI_REGS($28) LONG_S sp, TI_REGS($28) PTR_LA ra, ret_from_irq - j plat_irq_dispatch + PTR_LA v0, plat_irq_dispatch + jr v0 +#ifdef CONFIG_CPU_MICROMIPS + nop +#endif END(handle_int) __INIT @@ -222,11 +228,14 @@ NESTED(except_vec4, 0, sp) /* * EJTAG debug exception handler. * The EJTAG debug exception entry point is 0xbfc00480, which - * normally is in the boot PROM, so the boot PROM must do a + * normally is in the boot PROM, so the boot PROM must do an * unconditional jump to this vector. */ NESTED(except_vec_ejtag_debug, 0, sp) j ejtag_debug_handler +#ifdef CONFIG_CPU_MICROMIPS + nop +#endif END(except_vec_ejtag_debug) __FINIT @@ -251,9 +260,10 @@ NESTED(except_vec_vi, 0, sp) FEXPORT(except_vec_vi_mori) ori a0, $0, 0 #endif /* CONFIG_MIPS_MT_SMTC */ + PTR_LA v1, except_vec_vi_handler FEXPORT(except_vec_vi_lui) lui v0, 0 /* Patched */ - j except_vec_vi_handler + jr v1 FEXPORT(except_vec_vi_ori) ori v0, 0 /* Patched */ .set pop @@ -354,6 +364,9 @@ EXPORT(ejtag_debug_buffer) */ NESTED(except_vec_nmi, 0, sp) j nmi_handler +#ifdef CONFIG_CPU_MICROMIPS + nop +#endif END(except_vec_nmi) __FINIT @@ -500,13 +513,35 @@ NESTED(nmi_handler, PT_SIZE, sp) .set push .set noat .set noreorder - /* 0x7c03e83b: rdhwr v1,$29 */ + /* MIPS32: 0x7c03e83b: rdhwr v1,$29 */ + /* microMIPS: 0x007d6b3c: rdhwr v1,$29 */ MFC0 k1, CP0_EPC - lui k0, 0x7c03 - lw k1, (k1) - ori k0, 0xe83b - .set reorder +#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2) + and k0, k1, 1 + beqz k0, 1f + xor k1, k0 + lhu k0, (k1) + lhu k1, 2(k1) + ins k1, k0, 16, 16 + lui k0, 0x007d + b docheck + ori k0, 0x6b3c +1: + lui k0, 0x7c03 + lw k1, (k1) + ori k0, 0xe83b +#else + andi k0, k1, 1 + bnez k0, handle_ri + lui k0, 0x7c03 + lw k1, (k1) + ori k0, 0xe83b +#endif + .set reorder +docheck: bne k0, k1, handle_ri /* if not ours */ + +isrdhwr: /* The insn is rdhwr. No need to check CAUSE.BD here. */ get_saved_sp /* k1 := current_thread_info */ .set noreorder diff --git a/arch/mips/kernel/scall32-o32.S b/arch/mips/kernel/scall32-o32.S index 9ea29649fc28..9b36424b03c5 100644 --- a/arch/mips/kernel/scall32-o32.S +++ b/arch/mips/kernel/scall32-o32.S @@ -138,9 +138,18 @@ stackargs: 5: jr t1 sw t5, 16(sp) # argument #5 to ksp +#ifdef CONFIG_CPU_MICROMIPS sw t8, 28(sp) # argument #8 to ksp + nop sw t7, 24(sp) # argument #7 to ksp + nop sw t6, 20(sp) # argument #6 to ksp + nop +#else + sw t8, 28(sp) # argument #8 to ksp + sw t7, 24(sp) # argument #7 to ksp + sw t6, 20(sp) # argument #6 to ksp +#endif 6: j stack_done # go back nop .set pop diff --git a/arch/mips/kernel/smtc-asm.S b/arch/mips/kernel/smtc-asm.S index 76016ac0a9c8..2866863a39df 100644 --- a/arch/mips/kernel/smtc-asm.S +++ b/arch/mips/kernel/smtc-asm.S @@ -49,6 +49,9 @@ CAN WE PROVE THAT WE WON'T DO THIS IF INTS DISABLED?? .text .align 5 FEXPORT(__smtc_ipi_vector) +#ifdef CONFIG_CPU_MICROMIPS + nop +#endif .set noat /* Disable thread scheduling to make Status update atomic */ DMT 27 # dmt k1 diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 333782b83164..571a69c57d82 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -8,8 +8,8 @@ * Copyright (C) 1998 Ulf Carlsson * Copyright (C) 1999 Silicon Graphics, Inc. * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com - * Copyright (C) 2000, 01 MIPS Technologies, Inc. * Copyright (C) 2002, 2003, 2004, 2005, 2007 Maciej W. Rozycki + * Copyright (C) 2000, 2001, 2012 MIPS Technologies, Inc. All rights reserved. */ #include #include @@ -83,10 +83,6 @@ extern asmlinkage void handle_dsp(void); extern asmlinkage void handle_mcheck(void); extern asmlinkage void handle_reserved(void); -extern int fpu_emulator_cop1Handler(struct pt_regs *xcp, - struct mips_fpu_struct *ctx, int has_fpu, - void *__user *fault_addr); - void (*board_be_init)(void); int (*board_be_handler)(struct pt_regs *regs, int is_fixup); void (*board_nmi_handler_setup)(void); @@ -495,6 +491,12 @@ asmlinkage void do_be(struct pt_regs *regs) #define SYNC 0x0000000f #define RDHWR 0x0000003b +/* microMIPS definitions */ +#define MM_POOL32A_FUNC 0xfc00ffff +#define MM_RDHWR 0x00006b3c +#define MM_RS 0x001f0000 +#define MM_RT 0x03e00000 + /* * The ll_bit is cleared by r*_switch.S */ @@ -609,42 +611,62 @@ static int simulate_llsc(struct pt_regs *regs, unsigned int opcode) * Simulate trapping 'rdhwr' instructions to provide user accessible * registers not implemented in hardware. */ -static int simulate_rdhwr(struct pt_regs *regs, unsigned int opcode) +static int simulate_rdhwr(struct pt_regs *regs, int rd, int rt) { struct thread_info *ti = task_thread_info(current); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, + 1, regs, 0); + switch (rd) { + case 0: /* CPU number */ + regs->regs[rt] = smp_processor_id(); + return 0; + case 1: /* SYNCI length */ + regs->regs[rt] = min(current_cpu_data.dcache.linesz, + current_cpu_data.icache.linesz); + return 0; + case 2: /* Read count register */ + regs->regs[rt] = read_c0_count(); + return 0; + case 3: /* Count register resolution */ + switch (current_cpu_data.cputype) { + case CPU_20KC: + case CPU_25KF: + regs->regs[rt] = 1; + break; + default: + regs->regs[rt] = 2; + } + return 0; + case 29: + regs->regs[rt] = ti->tp_value; + return 0; + default: + return -1; + } +} + +static int simulate_rdhwr_normal(struct pt_regs *regs, unsigned int opcode) +{ if ((opcode & OPCODE) == SPEC3 && (opcode & FUNC) == RDHWR) { int rd = (opcode & RD) >> 11; int rt = (opcode & RT) >> 16; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, regs, 0); - switch (rd) { - case 0: /* CPU number */ - regs->regs[rt] = smp_processor_id(); - return 0; - case 1: /* SYNCI length */ - regs->regs[rt] = min(current_cpu_data.dcache.linesz, - current_cpu_data.icache.linesz); - return 0; - case 2: /* Read count register */ - regs->regs[rt] = read_c0_count(); - return 0; - case 3: /* Count register resolution */ - switch (current_cpu_data.cputype) { - case CPU_20KC: - case CPU_25KF: - regs->regs[rt] = 1; - break; - default: - regs->regs[rt] = 2; - } - return 0; - case 29: - regs->regs[rt] = ti->tp_value; - return 0; - default: - return -1; - } + + simulate_rdhwr(regs, rd, rt); + return 0; + } + + /* Not ours. */ + return -1; +} + +static int simulate_rdhwr_mm(struct pt_regs *regs, unsigned short opcode) +{ + if ((opcode & MM_POOL32A_FUNC) == MM_RDHWR) { + int rd = (opcode & MM_RS) >> 16; + int rt = (opcode & MM_RT) >> 21; + simulate_rdhwr(regs, rd, rt); + return 0; } /* Not ours. */ @@ -826,9 +848,29 @@ static void do_trap_or_bp(struct pt_regs *regs, unsigned int code, asmlinkage void do_bp(struct pt_regs *regs) { unsigned int opcode, bcode; - - if (__get_user(opcode, (unsigned int __user *) exception_epc(regs))) - goto out_sigsegv; + unsigned long epc; + u16 instr[2]; + + if (get_isa16_mode(regs->cp0_epc)) { + /* Calculate EPC. */ + epc = exception_epc(regs); + if (cpu_has_mmips) { + if ((__get_user(instr[0], (u16 __user *)msk_isa16_mode(epc)) || + (__get_user(instr[1], (u16 __user *)msk_isa16_mode(epc + 2))))) + goto out_sigsegv; + opcode = (instr[0] << 16) | instr[1]; + } else { + /* MIPS16e mode */ + if (__get_user(instr[0], (u16 __user *)msk_isa16_mode(epc))) + goto out_sigsegv; + bcode = (instr[0] >> 6) & 0x3f; + do_trap_or_bp(regs, bcode, "Break"); + return; + } + } else { + if (__get_user(opcode, (unsigned int __user *) exception_epc(regs))) + goto out_sigsegv; + } /* * There is the ancient bug in the MIPS assemblers that the break @@ -869,13 +911,22 @@ out_sigsegv: asmlinkage void do_tr(struct pt_regs *regs) { unsigned int opcode, tcode = 0; + u16 instr[2]; + unsigned long epc = exception_epc(regs); - if (__get_user(opcode, (unsigned int __user *) exception_epc(regs))) - goto out_sigsegv; + if ((__get_user(instr[0], (u16 __user *)msk_isa16_mode(epc))) || + (__get_user(instr[1], (u16 __user *)msk_isa16_mode(epc + 2)))) + goto out_sigsegv; + opcode = (instr[0] << 16) | instr[1]; /* Immediate versions don't provide a code. */ - if (!(opcode & OPCODE)) - tcode = ((opcode >> 6) & ((1 << 10) - 1)); + if (!(opcode & OPCODE)) { + if (get_isa16_mode(regs->cp0_epc)) + /* microMIPS */ + tcode = (opcode >> 12) & 0x1f; + else + tcode = ((opcode >> 6) & ((1 << 10) - 1)); + } do_trap_or_bp(regs, tcode, "Trap"); return; @@ -888,6 +939,7 @@ asmlinkage void do_ri(struct pt_regs *regs) { unsigned int __user *epc = (unsigned int __user *)exception_epc(regs); unsigned long old_epc = regs->cp0_epc; + unsigned long old31 = regs->regs[31]; unsigned int opcode = 0; int status = -1; @@ -900,23 +952,37 @@ asmlinkage void do_ri(struct pt_regs *regs) if (unlikely(compute_return_epc(regs) < 0)) return; - if (unlikely(get_user(opcode, epc) < 0)) - status = SIGSEGV; + if (get_isa16_mode(regs->cp0_epc)) { + unsigned short mmop[2] = { 0 }; - if (!cpu_has_llsc && status < 0) - status = simulate_llsc(regs, opcode); + if (unlikely(get_user(mmop[0], epc) < 0)) + status = SIGSEGV; + if (unlikely(get_user(mmop[1], epc) < 0)) + status = SIGSEGV; + opcode = (mmop[0] << 16) | mmop[1]; - if (status < 0) - status = simulate_rdhwr(regs, opcode); + if (status < 0) + status = simulate_rdhwr_mm(regs, opcode); + } else { + if (unlikely(get_user(opcode, epc) < 0)) + status = SIGSEGV; - if (status < 0) - status = simulate_sync(regs, opcode); + if (!cpu_has_llsc && status < 0) + status = simulate_llsc(regs, opcode); + + if (status < 0) + status = simulate_rdhwr_normal(regs, opcode); + + if (status < 0) + status = simulate_sync(regs, opcode); + } if (status < 0) status = SIGILL; if (unlikely(status > 0)) { regs->cp0_epc = old_epc; /* Undo skip-over. */ + regs->regs[31] = old31; force_sig(status, current); } } @@ -986,7 +1052,7 @@ static int default_cu2_call(struct notifier_block *nfb, unsigned long action, asmlinkage void do_cpu(struct pt_regs *regs) { unsigned int __user *epc; - unsigned long old_epc; + unsigned long old_epc, old31; unsigned int opcode; unsigned int cpid; int status; @@ -1000,26 +1066,41 @@ asmlinkage void do_cpu(struct pt_regs *regs) case 0: epc = (unsigned int __user *)exception_epc(regs); old_epc = regs->cp0_epc; + old31 = regs->regs[31]; opcode = 0; status = -1; if (unlikely(compute_return_epc(regs) < 0)) return; - if (unlikely(get_user(opcode, epc) < 0)) - status = SIGSEGV; + if (get_isa16_mode(regs->cp0_epc)) { + unsigned short mmop[2] = { 0 }; - if (!cpu_has_llsc && status < 0) - status = simulate_llsc(regs, opcode); + if (unlikely(get_user(mmop[0], epc) < 0)) + status = SIGSEGV; + if (unlikely(get_user(mmop[1], epc) < 0)) + status = SIGSEGV; + opcode = (mmop[0] << 16) | mmop[1]; - if (status < 0) - status = simulate_rdhwr(regs, opcode); + if (status < 0) + status = simulate_rdhwr_mm(regs, opcode); + } else { + if (unlikely(get_user(opcode, epc) < 0)) + status = SIGSEGV; + + if (!cpu_has_llsc && status < 0) + status = simulate_llsc(regs, opcode); + + if (status < 0) + status = simulate_rdhwr_normal(regs, opcode); + } if (status < 0) status = SIGILL; if (unlikely(status > 0)) { regs->cp0_epc = old_epc; /* Undo skip-over. */ + regs->regs[31] = old31; force_sig(status, current); } @@ -1333,7 +1414,7 @@ asmlinkage void cache_parity_error(void) void ejtag_exception_handler(struct pt_regs *regs) { const int field = 2 * sizeof(unsigned long); - unsigned long depc, old_epc; + unsigned long depc, old_epc, old_ra; unsigned int debug; printk(KERN_DEBUG "SDBBP EJTAG debug exception - not handled yet, just ignored!\n"); @@ -1348,10 +1429,12 @@ void ejtag_exception_handler(struct pt_regs *regs) * calculation. */ old_epc = regs->cp0_epc; + old_ra = regs->regs[31]; regs->cp0_epc = depc; - __compute_return_epc(regs); + compute_return_epc(regs); depc = regs->cp0_epc; regs->cp0_epc = old_epc; + regs->regs[31] = old_ra; } else depc += 4; write_c0_depc(depc); @@ -1392,9 +1475,24 @@ void __init *set_except_vector(int n, void *addr) unsigned long handler = (unsigned long) addr; unsigned long old_handler = exception_handlers[n]; +#ifdef CONFIG_CPU_MICROMIPS + /* + * Only the TLB handlers are cache aligned with an even + * address. All other handlers are on an odd address and + * require no modification. Otherwise, MIPS32 mode will + * be entered when handling any TLB exceptions. That + * would be bad...since we must stay in microMIPS mode. + */ + if (!(handler & 0x1)) + handler |= 1; +#endif exception_handlers[n] = handler; if (n == 0 && cpu_has_divec) { +#ifdef CONFIG_CPU_MICROMIPS + unsigned long jump_mask = ~((1 << 27) - 1); +#else unsigned long jump_mask = ~((1 << 28) - 1); +#endif u32 *buf = (u32 *)(ebase + 0x200); unsigned int k0 = 26; if ((handler & jump_mask) == ((ebase + 0x200) & jump_mask)) { @@ -1421,17 +1519,18 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) unsigned long handler; unsigned long old_handler = vi_handlers[n]; int srssets = current_cpu_data.srsets; - u32 *w; + u16 *h; unsigned char *b; BUG_ON(!cpu_has_veic && !cpu_has_vint); + BUG_ON((n < 0) && (n > 9)); if (addr == NULL) { handler = (unsigned long) do_default_vi; srs = 0; } else handler = (unsigned long) addr; - vi_handlers[n] = (unsigned long) addr; + vi_handlers[n] = handler; b = (unsigned char *)(ebase + 0x200 + n*VECTORSPACING); @@ -1450,9 +1549,8 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) if (srs == 0) { /* * If no shadow set is selected then use the default handler - * that does normal register saving and a standard interrupt exit + * that does normal register saving and standard interrupt exit */ - extern char except_vec_vi, except_vec_vi_lui; extern char except_vec_vi_ori, except_vec_vi_end; extern char rollback_except_vec_vi; @@ -1465,11 +1563,20 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) * Status.IM bit to be masked before going there. */ extern char except_vec_vi_mori; +#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_BIG_ENDIAN) + const int mori_offset = &except_vec_vi_mori - vec_start + 2; +#else const int mori_offset = &except_vec_vi_mori - vec_start; +#endif #endif /* CONFIG_MIPS_MT_SMTC */ - const int handler_len = &except_vec_vi_end - vec_start; +#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_BIG_ENDIAN) + const int lui_offset = &except_vec_vi_lui - vec_start + 2; + const int ori_offset = &except_vec_vi_ori - vec_start + 2; +#else const int lui_offset = &except_vec_vi_lui - vec_start; const int ori_offset = &except_vec_vi_ori - vec_start; +#endif + const int handler_len = &except_vec_vi_end - vec_start; if (handler_len > VECTORSPACING) { /* @@ -1479,30 +1586,44 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) panic("VECTORSPACING too small"); } - memcpy(b, vec_start, handler_len); + set_handler(((unsigned long)b - ebase), vec_start, +#ifdef CONFIG_CPU_MICROMIPS + (handler_len - 1)); +#else + handler_len); +#endif #ifdef CONFIG_MIPS_MT_SMTC BUG_ON(n > 7); /* Vector index %d exceeds SMTC maximum. */ - w = (u32 *)(b + mori_offset); - *w = (*w & 0xffff0000) | (0x100 << n); + h = (u16 *)(b + mori_offset); + *h = (0x100 << n); #endif /* CONFIG_MIPS_MT_SMTC */ - w = (u32 *)(b + lui_offset); - *w = (*w & 0xffff0000) | (((u32)handler >> 16) & 0xffff); - w = (u32 *)(b + ori_offset); - *w = (*w & 0xffff0000) | ((u32)handler & 0xffff); + h = (u16 *)(b + lui_offset); + *h = (handler >> 16) & 0xffff; + h = (u16 *)(b + ori_offset); + *h = (handler & 0xffff); local_flush_icache_range((unsigned long)b, (unsigned long)(b+handler_len)); } else { /* - * In other cases jump directly to the interrupt handler - * - * It is the handlers responsibility to save registers if required - * (eg hi/lo) and return from the exception using "eret" + * In other cases jump directly to the interrupt handler. It + * is the handler's responsibility to save registers if required + * (eg hi/lo) and return from the exception using "eret". */ - w = (u32 *)b; - *w++ = 0x08000000 | (((u32)handler >> 2) & 0x03fffff); /* j handler */ - *w = 0; + u32 insn; + + h = (u16 *)b; + /* j handler */ +#ifdef CONFIG_CPU_MICROMIPS + insn = 0xd4000000 | (((u32)handler & 0x07ffffff) >> 1); +#else + insn = 0x08000000 | (((u32)handler & 0x0fffffff) >> 2); +#endif + h[0] = (insn >> 16) & 0xffff; + h[1] = insn & 0xffff; + h[2] = 0; + h[3] = 0; local_flush_icache_range((unsigned long)b, (unsigned long)(b+8)); } @@ -1663,7 +1784,11 @@ void __cpuinit per_cpu_trap_init(bool is_boot_cpu) /* Install CPU exception handler */ void __cpuinit set_handler(unsigned long offset, void *addr, unsigned long size) { +#ifdef CONFIG_CPU_MICROMIPS + memcpy((void *)(ebase + offset), ((unsigned char *)addr - 1), size); +#else memcpy((void *)(ebase + offset), addr, size); +#endif local_flush_icache_range(ebase + offset, ebase + offset + size); } @@ -1697,8 +1822,9 @@ __setup("rdhwr_noopt", set_rdhwr_noopt); void __init trap_init(void) { - extern char except_vec3_generic, except_vec3_r4000; + extern char except_vec3_generic; extern char except_vec4; + extern char except_vec3_r4000; unsigned long i; int rollback; @@ -1831,11 +1957,11 @@ void __init trap_init(void) if (cpu_has_vce) /* Special exception: R4[04]00 uses also the divec space. */ - memcpy((void *)(ebase + 0x180), &except_vec3_r4000, 0x100); + set_handler(0x180, &except_vec3_r4000, 0x100); else if (cpu_has_4kex) - memcpy((void *)(ebase + 0x180), &except_vec3_generic, 0x80); + set_handler(0x180, &except_vec3_generic, 0x80); else - memcpy((void *)(ebase + 0x080), &except_vec3_generic, 0x80); + set_handler(0x080, &except_vec3_generic, 0x80); local_flush_icache_range(ebase, ebase + 0x400); flush_tlb_handlers(); diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index fb4ca9984ed1..0a68f2a4e690 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -2103,6 +2103,13 @@ static void __cpuinit build_r4000_tlb_load_handler(void) uasm_l_nopage_tlbl(&l, p); build_restore_work_registers(&p); +#ifdef CONFIG_CPU_MICROMIPS + if ((unsigned long)tlb_do_page_fault_0 & 1) { + uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0)); + uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0)); + uasm_i_jr(&p, K0); + } else +#endif uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); uasm_i_nop(&p); @@ -2150,6 +2157,13 @@ static void __cpuinit build_r4000_tlb_store_handler(void) uasm_l_nopage_tlbs(&l, p); build_restore_work_registers(&p); +#ifdef CONFIG_CPU_MICROMIPS + if ((unsigned long)tlb_do_page_fault_1 & 1) { + uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); + uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); + uasm_i_jr(&p, K0); + } else +#endif uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); uasm_i_nop(&p); @@ -2198,6 +2212,13 @@ static void __cpuinit build_r4000_tlb_modify_handler(void) uasm_l_nopage_tlbm(&l, p); build_restore_work_registers(&p); +#ifdef CONFIG_CPU_MICROMIPS + if ((unsigned long)tlb_do_page_fault_1 & 1) { + uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); + uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); + uasm_i_jr(&p, K0); + } else +#endif uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); uasm_i_nop(&p); diff --git a/arch/mips/mti-sead3/sead3-init.c b/arch/mips/mti-sead3/sead3-init.c index 562dccf56ee8..bfbd17b120a2 100644 --- a/arch/mips/mti-sead3/sead3-init.c +++ b/arch/mips/mti-sead3/sead3-init.c @@ -65,7 +65,41 @@ static void __init mips_nmi_setup(void) base = cpu_has_veic ? (void *)(CAC_BASE + 0xa80) : (void *)(CAC_BASE + 0x380); +#ifdef CONFIG_CPU_MICROMIPS + /* + * Decrement the exception vector address by one for microMIPS. + */ + memcpy(base, (&except_vec_nmi - 1), 0x80); + + /* + * This is a hack. We do not know if the boot loader was built with + * microMIPS instructions or not. If it was not, the NMI exception + * code at 0x80000a80 will be taken in MIPS32 mode. The hand coded + * assembly below forces us into microMIPS mode if we are a pure + * microMIPS kernel. The assembly instructions are: + * + * 3C1A8000 lui k0,0x8000 + * 375A0381 ori k0,k0,0x381 + * 03400008 jr k0 + * 00000000 nop + * + * The mode switch occurs by jumping to the unaligned exception + * vector address at 0x80000381 which would have been 0x80000380 + * in MIPS32 mode. The jump to the unaligned address transitions + * us into microMIPS mode. + */ + if (!cpu_has_veic) { + void *base2 = (void *)(CAC_BASE + 0xa80); + *((unsigned int *)base2) = 0x3c1a8000; + *((unsigned int *)base2 + 1) = 0x375a0381; + *((unsigned int *)base2 + 2) = 0x03400008; + *((unsigned int *)base2 + 3) = 0x00000000; + flush_icache_range((unsigned long)base2, + (unsigned long)base2 + 0x10); + } +#else memcpy(base, &except_vec_nmi, 0x80); +#endif flush_icache_range((unsigned long)base, (unsigned long)base + 0x80); } @@ -76,7 +110,21 @@ static void __init mips_ejtag_setup(void) base = cpu_has_veic ? (void *)(CAC_BASE + 0xa00) : (void *)(CAC_BASE + 0x300); +#ifdef CONFIG_CPU_MICROMIPS + /* Deja vu... */ + memcpy(base, (&except_vec_ejtag_debug - 1), 0x80); + if (!cpu_has_veic) { + void *base2 = (void *)(CAC_BASE + 0xa00); + *((unsigned int *)base2) = 0x3c1a8000; + *((unsigned int *)base2 + 1) = 0x375a0301; + *((unsigned int *)base2 + 2) = 0x03400008; + *((unsigned int *)base2 + 3) = 0x00000000; + flush_icache_range((unsigned long)base2, + (unsigned long)base2 + 0x10); + } +#else memcpy(base, &except_vec_ejtag_debug, 0x80); +#endif flush_icache_range((unsigned long)base, (unsigned long)base + 0x80); } -- cgit v1.2.3